package com.cnap.multiclouldmgmt.service.impl;

import cn.hutool.core.lang.UUID;
import cn.hutool.core.util.IdUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.cnap.multiclouldmgmt.model.RepByPage;
import com.cnap.multiclouldmgmt.model.alarm.*;
import com.cnap.multiclouldmgmt.model.enums.K8sRoleEnum;
import com.cnap.multiclouldmgmt.model.enums.K8sStatusEnum;
import com.cnap.multiclouldmgmt.model.enums.NodeTypeEnum;
import com.cnap.multiclouldmgmt.model.gpu.FlavorGPUInfoWithImageDisk;
import com.cnap.multiclouldmgmt.model.gpu.GpuSpecInfo;
import com.cnap.multiclouldmgmt.model.jmeter.JmeterDTO;
import com.cnap.multiclouldmgmt.model.k8sModels.*;
import com.cnap.multiclouldmgmt.model.k8sMonitorModels.SvcMonitorModel;
import com.cnap.multiclouldmgmt.model.knowledge.ErrorInjectModel;
import com.cnap.multiclouldmgmt.model.knowledge.MicroSvcPressParamModel;
import com.cnap.multiclouldmgmt.model.log.OperationTypeEnum;
import com.cnap.multiclouldmgmt.model.microSvc.*;
import com.cnap.multiclouldmgmt.model.vm.VmModel;
import com.cnap.multiclouldmgmt.repository.cnap.AlarmMapper;
import com.cnap.multiclouldmgmt.repository.cnap.MicroSvcMapper;
import com.cnap.multiclouldmgmt.repository.master.K8sClusterEventsMapper;
import com.cnap.multiclouldmgmt.repository.master.K8sClusterInfoMapper;
import com.cnap.multiclouldmgmt.repository.master.MicroSvcUseInfoMapper;
import com.cnap.multiclouldmgmt.service.*;
import com.cnap.multiclouldmgmt.util.CnapCloudMgmtConstants;
import com.cnap.multiclouldmgmt.util.DateUtils;
import com.cnap.multiclouldmgmt.util.OtherUtils;
import com.cnap.multiclouldmgmt.util.QuantityUtils;
import io.fabric8.kubernetes.api.model.*;
import io.fabric8.kubernetes.api.model.apps.*;
import io.fabric8.kubernetes.api.model.metrics.v1beta1.ContainerMetrics;
import io.fabric8.kubernetes.api.model.metrics.v1beta1.NodeMetrics;
import io.fabric8.kubernetes.api.model.metrics.v1beta1.PodMetrics;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.utils.Serialization;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.ByteArrayInputStream;
import java.io.UnsupportedEncodingException;
import java.text.ParseException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;

/**
 * @author dyq
 *
 * 可以提供
 * （1）集群数据：
 * 集群的状态、
 * CPU总量、CPU使用量、
 * 内存总量、内存使用量、
 * 硬盘总量、硬盘使用量、硬盘吞吐量、
 * 网络带宽、网络延时、网络抖动。
 * （2）节点数据：
 * CPU核数、CPU使用数、
 * 内存总量、
 * 硬盘总量、硬盘使用量、硬盘吞吐量、
 * 网络带宽、
 * 节点上部署的Pod数量。
 * （3）应用数据：
 * Pod数量、Pod对应的容器组，
 * Pod状态，
 * 应用运行时长,应用采用的镜像文件，
 * 配置文件。
 * （4）Pod数据：
 * Pod CPU使用量，Pod CPU最大使用限制、
 * Pod内存使用量、Pod内存最大使用限制、
 * Pod容器组状态、Pod容器组镜像、运行时长、
 * 重启次数，Pod配置文件。
 * （5）Service数据：
 * service的IP、service类型，
 * service的selector，service对应的Pod数量和状态、
 * service配置文件。
 *
 * 
 * kubectl常用命令
 * # 显示合并后的 kubeconfig 配置
 * $ kubectl config view
 * # 同时使用多个 kubeconfig 文件并查看合并后的配置
 * $ KUBECONFIG=~/.kube/config:~/.kube/kubconfig2 kubectl config view
 * # 获取 e2e 用户的密码
 * $ kubectl config view -o jsonpath='{.users[?(@.name == "e2e")].user.password}'
 * # 显示当前的上下文
 * $ kubectl config current-context
 * # 设置默认上下文为 my-cluster-name
 * $ kubectl config use-context my-cluster-name
 * # 向 kubeconf 中增加支持基本认证的新集群
 * $ kubectl config set-credentials kubeuser/foo.kubernetes.com --username=kubeuser --password=kubepassword
 * # 使用指定的用户名和 namespace 设置上下文
 * $ kubectl config set-context gce --user=cluster-admin --namespace=foo \
 *   && kubectl config use-context gce
 *
 * # 创建资源
 * $ kubectl create -f ./my-manifest.yaml
 * # 使用多个文件创建资源
 * $ kubectl create -f ./my1.yaml -f ./my2.yaml
 * # 使用目录下的所有清单文件来创建资源
 * $ kubectl create -f ./dir
 * # 使用 url 来创建资源
 * $ kubectl create -f https://git.io/vPieo
 * # 启动一个 nginx 实例
 * $ kubectl run nginx --image=nginx
 * # 获取 pod 和 svc 的文档
 * $ kubectl explain pods,svc
 * # Get commands with basic output
 * # 列出所有 namespace 中的所有 service
 * $ kubectl get services
 *
 * # 列出所有 namespace 中的所有 pod
 * $ kubectl get pods --all-namespaces
 *
 * # 列出所有 pod 并显示详细信息
 * $ kubectl get pods -o wide
 *
 * # 列出指定 deployment
 * $ kubectl get deployment my-dep
 *
 * # 列出该 namespace 中的所有 pod 包括未初始化的
 * $ kubectl get pods --include-uninitialized
 *
 * # 使用详细输出来描述命令
 * $ kubectl describe nodes my-node
 * $ kubectl describe pods my-pod
 *
 * # List Services Sorted by Name
 * $ kubectl get services --sort-by=.metadata.name
 *
 * # 根据重启次数排序列出 pod
 * $ kubectl get pods --sort-by='.status.containerStatuses[0].restartCount'
 *
 * # 获取所有具有 app=cassandra 的 pod 中的 version 标签
 * $ kubectl get pods --selector=app=cassandra rc -o \
 *   jsonpath='{.items[*].metadata.labels.version}'
 *
 * # 获取所有节点的 ExternalIP
 * $ kubectl get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="ExternalIP")].address}'
 *
 * # 列出属于某个 PC 的 Pod 的名字
 * # “jq”命令用于转换复杂的 jsonpath，参考 https://stedolan.github.io/jq/
 * $ sel=${$(kubectl get rc my-rc --output=json | jq -j '.spec.selector | to_entries | .[] | "\(.key)=\(.value),"')%?}
 * $ echo $(kubectl get pods --selector=$sel --output=jsonpath={.items..metadata.name})
 *
 * # 查看哪些节点已就绪
 * $ JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' \
 *  && kubectl get nodes -o jsonpath="$JSONPATH" | grep "Ready=True"
 *
 * # 列出当前 Pod 中使用的 Secret
 * $ kubectl get pods -o json | jq '.items[].spec.containers[].env[]?.valueFrom.secretKeyRef.name' | grep -v null | sort | uniq
 *
 *$ kubectl rolling-update frontend-v1 -f frontend-v2.json           # 滚动更新 pod frontend-v1
 * $ kubectl rolling-update frontend-v1 frontend-v2 --image=image:v2  # 更新资源名称并更新镜像
 * $ kubectl rolling-update frontend --image=image:v2                 # 更新 frontend pod 中的镜像
 * $ kubectl rolling-update frontend-v1 frontend-v2 --rollback        # 退出已存在的进行中的滚动更新
 * $ cat pod.json | kubectl replace -f -                              # 基于 stdin 输入的 JSON 替换 pod
 * # 强制替换，删除后重新创建资源。会导致服务中断。
 * $ kubectl replace --force -f ./pod.json
 * # 为 nginx RC 创建服务，启用本地 80 端口连接到容器上的 8000 端口
 * $ kubectl expose rc nginx --port=80 --target-port=8000
 * # 更新单容器 pod 的镜像版本（tag）到 v4
 * $ kubectl get pod mypod -o yaml | sed 's/\(image: myimage\):.*$/\1:v4/' | kubectl replace -f -
 * # 添加标签
 * $ kubectl label pods my-pod new-label=awesome
 *  # 添加注解
 * $ kubectl annotate pods my-pod icon-url=http://goo.gl/XXBTWq
 * # 自动扩展 deployment “foo”
 * $ kubectl autoscale deployment foo --min=2 --max=10
 *
 * **
# 标记 my-node 不可调度
$ kubectl cordon my-node
# 清空 my-node 以待维护
$ kubectl drain my-node
# 标记 my-node 可调度
$ kubectl uncordon my-node
# 显示 my-node 的指标度量
$ kubectl top node my-node
$ kubectl cluster-info
# 将当前集群状态输出到 stdout
$ kubectl cluster-info dump
# 将当前集群状态输出到 /path/to/cluster-state
$ kubectl cluster-info dump --output-directory=/path/to/cluster-state
# 如果该键和影响的污点（taint）已存在，则使用指定的值替换
$ kubectl taint nodes foo dedicated=special-user:NoSchedule
 */
@Service("K8sClusterServiceImpl")
public class K8sClusterServiceImpl implements K8sClusterService {

    private final static Logger LOGGER = LoggerFactory.getLogger(CacheServiceImpl.class);

    @Autowired
    private MultiCloudMgmtService multiCloudMgmtService;

    @Autowired
    private CacheService cacheService;

    @Autowired
    private K8sClusterInfoMapper k8sClusterInfoMapper;

    @Autowired
    private PrometheusService prometheusService;

    @Autowired
    private K8sClusterEventsMapper k8sClusterEventsMapper;

    @Value("${agent.image}")
    private  String agentImage;

    @Value("${agent.adminUrl}")
    private  String agentAdminUrl;

    @Autowired
    private CnapSshService cnapSshService;

    @Autowired
    private VmResourceService vmResourceService;

    @Autowired
    private OperationLogService operationLogService;

    @Autowired
    private MicroSvcMapper microSvcMapper;

    @Autowired
    private MicroSvcUseInfoMapper microSvcUseInfoMapper;


    @Autowired
    private AlarmMapper alarmMapper;

    @Autowired
    private JmeterService jmeterService;


    /**
     *
     * @param k8sClusterBasicModel
     * @param nodeName
     * @param labelKey
     * @param labelValue
     * @return
     * @throws ParseException
     */
    @Override
    public List<NodeModel> getAllNodeInfo(K8sClusterBasicModel k8sClusterBasicModel, String nodeName, String labelKey, String labelValue) throws Exception {

        List<NodeModel> nodeModels = new ArrayList<>();

        if (!StringUtils.equals(k8sClusterBasicModel.getK8sStatus(),K8sStatusEnum.Running.getChineseDescribe())){
            return nodeModels;
        }

        //获取pod的所有情况
        List<PodModel> podModels = getAllPodsInfo(k8sClusterBasicModel,null);
        //按nodename去分类pod,
        Map<String,List<PodModel>> nodePodeInfoMap = new HashMap<>();
        for (PodModel podModel:podModels){

            nodePodeInfoMap.computeIfAbsent(podModel.getNodeName(), k -> new ArrayList<>());

            nodePodeInfoMap.get(podModel.getNodeName()).add(podModel);
        }

        List<Node> nodes = new ArrayList<>();
        if(StringUtils.isEmpty(nodeName)) {
            nodes = k8sClusterBasicModel.getK8sClient().nodes().list().getItems();

        }else{

            if (k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName) != null) {
                nodes.add(k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName).get());
            }
        }

        //用来查找vm，vm的key是内网ip
        Map<String,VmModel> vmModelMap = new HashMap<>();
        if((k8sClusterBasicModel.getVmModels() != null)&&(!k8sClusterBasicModel.getVmModels().isEmpty()))
        for (VmModel vmModel:k8sClusterBasicModel.getVmModels()){
            vmModelMap.put(vmModel.getInnerIp(),vmModel);
        }

        for(Node node:nodes){

            if ((node == null)||(node.getMetadata() == null)){
                continue;
            }

            NodeModel nodeModel = new NodeModel();

            nodeModel.setK8sClusterIp(k8sClusterBasicModel.getK8sClusterIp());
            nodeModel.setK8sClusterLocation(k8sClusterBasicModel.getK8sClusterLocation());
            nodeModel.setK8sClusterName(k8sClusterBasicModel.getK8sClusterName());
            nodeModel.setCloudVendor(k8sClusterBasicModel.getCloudVendor());
            nodeModel.setCreateUser(k8sClusterBasicModel.getCreateUser());

            //节点名称
            nodeModel.setNodeName(node.getMetadata().getName());

            nodePodeInfoMap.computeIfAbsent(nodeModel.getNodeName(), k -> new ArrayList<>());
            nodeModel.setPodModels(nodePodeInfoMap.get(nodeModel.getNodeName()));

            //根据condition判断
            if (node.getStatus().getConditions() != null) {
                for (NodeCondition nodeCondition:node.getStatus().getConditions()) {
                    if (StringUtils.equals(nodeCondition.getType(),"Ready")){
                        if (StringUtils.equals(nodeCondition.getStatus(),"True")) {
                            nodeModel.setStatus(K8sStatusEnum.Ready.getDescribe());
                        }else {
                            nodeModel.setStatus(K8sStatusEnum.Error.getDescribe());
                        }
                    }
                }
            }
            //节点状态异常，就不获取其他东西
            if (StringUtils.equals(nodeModel.getStatus(),K8sStatusEnum.Error.getDescribe())){
                continue;
            }

            //节点类型，靠编辑
            if (node.getMetadata().getAnnotations().containsKey(CnapCloudMgmtConstants.CNAP_NODE_TYPE)){
                nodeModel.setNodeType(NodeTypeEnum.getChineseDescribeByDescribe(
                        node.getMetadata().getAnnotations().get(CnapCloudMgmtConstants.CNAP_NODE_TYPE)));
            }else {

                nodeModel.setNodeType(null);
            }


            //获取label
            Map<String,String> labelMap = new HashMap<>();
            if (node.getMetadata().getLabels() != null){
                labelMap = node.getMetadata().getLabels();
            }
            nodeModel.setNodeLabelMap(labelMap);

            //筛选label
            if (StringUtils.isNotEmpty(labelKey)){
                if (!labelMap.containsKey(labelKey)){
                    continue;
                }

                if (!StringUtils.equals(labelValue,labelMap.get(labelKey))){
                    continue;
                }
            }

            //节点的位置如果没编辑就和集群一致
            if (node.getMetadata().getAnnotations().containsKey(CnapCloudMgmtConstants.CNAP_NODE_LOCATION)){
                nodeModel.setLocation(node.getMetadata()
                        .getAnnotations().get(CnapCloudMgmtConstants.CNAP_NODE_LOCATION));
            }else {

                nodeModel.setLocation(k8sClusterBasicModel.getK8sClusterLocation());
            }


            //查看label的角色
            if (node.getMetadata().getLabels().containsKey("node-role.kubernetes.io/master")) {
                nodeModel.setNodeRole(K8sRoleEnum.Master.getStatus());
            }else{
                nodeModel.setNodeRole(K8sRoleEnum.Slave.getStatus());
            }

            //架构
            if (node.getMetadata().getAnnotations().containsKey(CnapCloudMgmtConstants.CNAP_NODE_FRAMEWORK)){
                nodeModel.setFramework(node.getMetadata()
                        .getAnnotations().get(CnapCloudMgmtConstants.CNAP_NODE_FRAMEWORK));
            }else {

                nodeModel.setFramework(node.getStatus().getNodeInfo().getArchitecture());
            }

            //获取节点对应的虚拟机名称
            nodeModel.setVmName(node.getMetadata()
                    .getAnnotations().getOrDefault(CnapCloudMgmtConstants.CNAP_NODE_VMNAME, nodeName));

            //ip
            List<NodeAddress> nodeAddresseList = node.getStatus().getAddresses();
            for (NodeAddress nodeAddress:nodeAddresseList) {
                if (StringUtils.contains(nodeAddress.getType(),"IP")) {
                    nodeModel.setNodeIp(nodeAddress.getAddress());
                    break;
                }
            }
            if (vmModelMap.containsKey(nodeModel.getNodeIp())){
                nodeModel.setNodeExtranlIp(vmModelMap.get(nodeModel.getNodeIp()).getExternalIp());
                nodeModel.setNodeInnerIp(vmModelMap.get(nodeModel.getNodeIp()).getInnerIp());
            }

            //容器总额度
            Map<String, Quantity> capacityMap = node.getStatus().getCapacity();
            if (capacityMap.get("pods") != null){
                nodeModel.setNodeContainerQuota(Integer.parseInt(capacityMap.get("pods").getAmount()));
            }else {
                nodeModel.setNodeContainerQuota(110);
            }

            //node上面在用着的额度
            nodeModel.setNodeContainerUseNum(nodePodeInfoMap.get(nodeModel.getNodeName()).size());

            //cpu总额度
            if(capacityMap.get("cpu") != null){

                long cpuTotal = QuantityUtils.countCpuUseToN(capacityMap.get("cpu"));
                nodeModel.setNodeCpuQuota(QuantityUtils.changeCpuNtoNum(cpuTotal));
                nodeModel.setNodeCpuQuotaByN(cpuTotal);
            }

            //cpu使用中
            long cpuUse = 0L;
            long memUse = 0L;
            //统计在正常状态中的pod数量
            int nomalPodNum = 0;
            for(PodModel podModel :nodePodeInfoMap.get(nodeModel.getNodeName())){
                cpuUse += Math.max(podModel.getCpuUseByN(),podModel.getPodReqCpuByN());
                memUse += Math.max(podModel.getMemUseByKi(),podModel.getPodReqMemByKi());

                if (StringUtils.equals(podModel.getStatus(),CnapCloudMgmtConstants.POD_STATUS_RUNNING)){
                    nomalPodNum++;
                }
            }
            nodeModel.setNodeCpuUseByN(cpuUse);
            nodeModel.setNodeCpuUseNum(QuantityUtils.changeCpuNtoNumStr(cpuUse));
            nodeModel.setNodeContainerNormalNum(nomalPodNum);

            //内存总额度
            if(capacityMap.get("memory") != null){

                long memTotal = QuantityUtils.countMemUseToKi(capacityMap.get("memory"));
                nodeModel.setNodeMemQuota(QuantityUtils.changeMemKitoGi(memTotal));
                nodeModel.setNodeMemQuotaByKi(memTotal);
            }

            //内存总使用
            nodeModel.setNodeMemUseNum(QuantityUtils.changeMemKiToGiStr(memUse));
            nodeModel.setNodeMemUseNumByKi(memUse);

            //获取metric
            if((k8sClusterBasicModel.getK8sClient().top() != null)
                    &&(k8sClusterBasicModel.getK8sClient().top().nodes()!= null)){
                NodeMetrics nodeMetrics =
                        k8sClusterBasicModel.getK8sClient().top().nodes().metrics(nodeModel.getNodeName());
                if ((nodeMetrics != null)
                        && (nodeMetrics.getUsage() != null)
                        && (!nodeMetrics.getUsage().isEmpty())) {

                    Quantity cpuQuantity = nodeMetrics.getUsage().get("cpu");
                    if (cpuQuantity != null) {
                        nodeModel.setNodeCpuUseByN(QuantityUtils.countCpuUseToN(cpuQuantity));
                        nodeModel.setNodeCpuUseNum(
                                QuantityUtils.changeCpuNtoNumStr(nodeModel.getNodeCpuUseByN()));
                    }

                    Quantity memQuantity = nodeMetrics.getUsage().get("memory");
                    if (memQuantity != null) {
                        nodeModel.setNodeMemUseNumByKi(QuantityUtils.countMemUseToKi(memQuantity));
                        nodeModel.setNodeMemUseNum(QuantityUtils.changeMemKiToGiStr(nodeModel.getNodeMemUseNumByKi()));
                    }

                }
            }

            //硬盘额度
            long diskTotal = 0L;
            if(capacityMap.get("ephemeral-storage") != null){

                diskTotal = QuantityUtils.countMemUseToKi(capacityMap.get("ephemeral-storage"));
                nodeModel.setNodeSysDiskQuota(QuantityUtils.changeMemKitoGi(diskTotal));
                nodeModel.setNodeDiskQuota(nodeModel.getNodeSysDiskQuota());

            }

            //硬盘已使用
            int diskFree = 0;
            long nodeNetIops = 0l;
            int diskIops = 0;
            int diskThroughput = 0;
            int netBandwidth = 0;
            try {
                //先去Prometheus获取硬盘剩余
                diskFree = QuantityUtils.changeDiskBtoGB(
                        prometheusService.getNodeDiskFreeUnitB(
                                k8sClusterBasicModel.getK8sClusterIp(), nodeModel.getNodeIp(),nodeModel.getArch()));

                //去Prometheus获取node的iops
                nodeNetIops = prometheusService.getNodeNetIOPS(k8sClusterBasicModel.getK8sClusterIp(),nodeModel.getNodeIp(),nodeModel.getArch());

                //获取节点的disk信息
                diskIops = (int)prometheusService.getNodeDiskIOPS(k8sClusterBasicModel.getK8sClusterIp()
                        ,nodeModel.getNodeIp(),nodeModel.getArch());
                diskThroughput = (int)prometheusService.getNodeDiskThroughputByMbPerS(k8sClusterBasicModel.getK8sClusterIp(),
                        nodeModel.getNodeIp(),nodeModel.getArch());

                netBandwidth = (int)(prometheusService.getNodeNetBandwidthBybps(k8sClusterBasicModel.getK8sClusterIp(),
                        nodeModel.getNodeIp(),nodeModel.getArch())/1024/1024);
            }catch (Exception e){
                LOGGER.error("getAllNodeInfo getSysNodeDisFree fail ip={}",k8sClusterBasicModel.getK8sClusterIp(),e);
            }
            nodeModel.setNodeSysDiskUseNum(Math.max(0,nodeModel.getNodeSysDiskQuota() - diskFree));
            nodeModel.setNodeDiskUseNum(nodeModel.getNodeSysDiskUseNum());
            nodeModel.setNodeNetIops(nodeNetIops);

            nodeModel.setDiskIops(diskIops);
            nodeModel.setDiskThroughput(diskThroughput);
            nodeModel.setNetBandwidth(netBandwidth);

            //创建时间
            nodeModel.setCreateTime(
                    DateUtils.getDateByYYMMDDTHHMMSS(node.getMetadata().getCreationTimestamp()));

            int nodeDiskTotal = 0;
            try{

                nodeDiskTotal = QuantityUtils.changeDiskBtoGB(
                        prometheusService.getNodeDiskTotalToB(
                                k8sClusterBasicModel.getK8sClusterIp(), nodeModel.getNodeIp(),nodeModel.getArch()));

                if (nodeDiskTotal > (nodeModel.getNodeSysDiskQuota()+5)){
                    nodeModel.setNodeDiskQuota(nodeDiskTotal - nodeModel.getNodeSysDiskQuota());
                }

            }catch (Exception e){
                LOGGER.error("getAllNodeInfo getNodeDisFree fail ip={}",k8sClusterBasicModel.getK8sClusterIp(),e);
            }
            
            //获取gpu相关的数据
            if(StringUtils.equalsIgnoreCase(nodeModel.getNodeHardware(),CnapCloudMgmtConstants.ARCH_GPU)){

                if (k8sClusterBasicModel.getGpuFlavorInfo() != null){

                    GpuSpecInfo gpuSpec
                            = JSONObject.parseObject(k8sClusterBasicModel.getGpuFlavorInfo().getGpuSpecStr(),GpuSpecInfo.class);

                        nodeModel.setNodeGpuMemStr(gpuSpec.getGpuMemorySize());
                        nodeModel.setNodeGpuSpec(gpuSpec.getGpuMemorySize());

                }

                nodeModel.setNodeGpuUsage(prometheusService.getGpuUsage(k8sClusterBasicModel.getK8sClusterIp()));
                nodeModel.setNodeGpuPower(prometheusService.getGpuPower(k8sClusterBasicModel.getK8sClusterIp()));
                nodeModel.setNodeGpuMemUse(prometheusService.getGpuMemUse(k8sClusterBasicModel.getK8sClusterIp()));
                nodeModel.setNodeGpuTemp(prometheusService.getGpuTemp(k8sClusterBasicModel.getK8sClusterIp()));
                double gpuMemFree = prometheusService.getGpuMemFree(k8sClusterBasicModel.getK8sClusterIp());
                nodeModel.setNodeGpuMemTotal(gpuMemFree + nodeModel.getNodeGpuMemUse());
                nodeModel.setNodeGpuMemUsage(Double.parseDouble(String.format("%.3f",
                        nodeModel.getNodeGpuMemUse()/nodeModel.getNodeGpuMemTotal())));

            }

            nodeModels.add(nodeModel);
        }

        return nodeModels;
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @return
     * @throws ParseException
     */
    @Override
    public List<NodeModel> getAllNodeInfo(K8sClusterBasicModel k8sClusterBasicModel) throws Exception {

        return getAllNodeInfo(k8sClusterBasicModel,null,null,null);
    }

    /**
     *
     * @param nodeModel
     * @throws Exception
     */
    @Override
    public void editK8sClusterNodeInfo(NodeModel nodeModel) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByIp(nodeModel.getK8sClusterIp());
        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        Node node = k8sClient.nodes().withName(nodeModel.getNodeName()).get();
        Map<String, String> annotationsMap = node.getMetadata().getAnnotations();

        /*原有设置node的方法
        if (StringUtils.isNotEmpty(nodeModel.getNodeType())) {
            annotationsMap.put(CnapCloudMgmtConstants.PML_NODE_TYPE,nodeModel.getNodeType());
        }

        if (StringUtils.isNotEmpty(nodeModel.getLocation())){
            annotationsMap.put(CnapCloudMgmtConstants.PML_NODE_LOCATION,nodeModel.getLocation());
        }

        if (StringUtils.isNotEmpty(nodeModel.getFramework())){
            annotationsMap.put(CnapCloudMgmtConstants.PML_NODE_FRAMEWORK,nodeModel.getFramework());
        }

        node.getMetadata().setAnnotations(annotationsMap);

         */

        node.getMetadata().setLabels(nodeModel.getNodeLabelMap());

        k8sClient.nodes().resource(node).createOrReplace();
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @return
     * @throws Exception
     */
    @Override
    public int getK8sClusterMasterNodeNum(String cloudVendor, String k8sClusterName) throws Exception {

        int result = 0;

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        List<Node> nodes = k8sClusterBasicModel.getK8sClient().nodes().list().getItems();

        for (Node node:nodes) {

            if (node.getMetadata() == null){
                continue;
            }
            //节点类型，靠编辑
            if (node.getMetadata().getLabels().containsKey("node-role.kubernetes.io/master")) {
                result++;
            }

        }

        return result;
    }


    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @return
     * @throws Exception
     */
    @Override
    public int getK8sClusterNodeNum(String cloudVendor, String k8sClusterName) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        return getK8sClusterNodeNum(k8sClusterBasicModel);
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @return
     * @throws Exception
     */
    @Override
    public int getK8sClusterNodeNum(K8sClusterBasicModel k8sClusterBasicModel) throws Exception {
        return k8sClusterBasicModel.getK8sClient().nodes().list().getItems().size();

    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @return
     * @throws Exception
     */
    @Override
    public int getK8sClusterSvcNum(String cloudVendor, String k8sClusterName) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        return k8sClusterBasicModel.getK8sClient().services().list().getItems().size();

    }
    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param nodeName
     * @param keyword
     * @param pageSize
     * @param pageNo
     * @param labelKey
     * @param labelValue
     * @param accountName
     * @return
     * @throws Exception
     */
    public RepByPage<NodeModel> getK8sClusterNodeInfosByConditions(String cloudVendor,
                                                                   String k8sClusterName, String nodeName,
                                                                   String keyword, int pageSize, int pageNo,
                                                                   String labelKey, String labelValue,String accountName) throws Exception {

        if ((StringUtils.isNotEmpty(labelValue))
                &&(StringUtils.isEmpty(labelKey))){
            labelKey = CnapCloudMgmtConstants.NODE_APP_TYPE_LABEL + labelValue.toLowerCase();
        }

        if(StringUtils.isNotEmpty(labelValue)) {
            labelValue = labelValue.toLowerCase();
        }

        List<K8sClusterBasicModel> k8sClusterBasicModels
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName,accountName);
        if ((k8sClusterBasicModels == null)||(k8sClusterBasicModels.isEmpty())){
            RepByPage<NodeModel> result = new RepByPage<>();
            result.setTotalNum(0);
            result.setPageNo(pageNo);
            result.setPageSize(pageSize);
            result.setResult(new ArrayList<>());
            return result;
        }
        List<NodeModel> allNodeModels = new ArrayList<>();
        for (K8sClusterBasicModel k8sClusterBasicModel:k8sClusterBasicModels) {
            allNodeModels.addAll(getAllNodeInfo(k8sClusterBasicModel, nodeName, labelKey, labelValue));
        }

        if (StringUtils.isNotEmpty(keyword)){
            List<NodeModel> nodeModelsByKeyWord = new ArrayList<>();
            for (NodeModel nodeModel:allNodeModels){

                if (StringUtils.contains(nodeModel.getNodeName(),keyword)
                        ||StringUtils.contains(nodeModel.getNodeRole(),keyword)
                        ||StringUtils.contains(nodeModel.getStatus(),keyword)
                        ||StringUtils.contains(nodeModel.getFramework(),keyword)
                        ||StringUtils.contains(nodeModel.getNodeIp(),keyword)
                        ||StringUtils.contains(nodeModel.getLocation(),keyword)
                        ||StringUtils.contains(nodeModel.getNodeType(),keyword)){

                    nodeModelsByKeyWord.add(nodeModel);
                }
            }

            return OtherUtils.getResultFromList(nodeModelsByKeyWord,pageNo,pageSize);
        }

        return OtherUtils.getResultFromList(allNodeModels,pageNo,pageSize);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param nodeName
     * @param keyword
     * @param pageSize
     * @param pageNo
     * @param labelKey
     * @param labelValue
     * @return
     * @throws Exception
     */
    @Override
    public RepByPage<NodeModel> getK8sClusterNodeInfosByConditions(String cloudVendor,
                                                                   String k8sClusterName, String nodeName,
                                                                   String keyword, int pageSize, int pageNo,
                                                                   String labelKey, String labelValue) throws Exception {

        return getK8sClusterNodeInfosByConditions(cloudVendor,
                k8sClusterName, nodeName,
                keyword, pageSize, pageNo, labelKey, labelValue,null);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param nodeName
     * @param keyword
     * @param pageSize
     * @param pageNo
     * @return
     * @throws Exception
     */
    @Override
    public RepByPage<NodeModel> getK8sClusterNodeInfosByConditions(String cloudVendor, String k8sClusterName,
                                                                   String nodeName, String keyword,
                                                                   int pageSize, int pageNo) throws Exception {
        return getK8sClusterNodeInfosByConditions(cloudVendor,k8sClusterName,nodeName,
                keyword,pageSize,pageNo,null,null);
    }

    @Override
    public RepByPage<NodeModel> getK8sClusterNodeInfosByConditions(String cloudVendor, String k8sClusterName, String keyword, int pageSize, int pageNo, String accountName) throws Exception {
        return getK8sClusterNodeInfosByConditions(cloudVendor,k8sClusterName,null,
                keyword,pageSize,pageNo,null,null,accountName);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param keyword
     * @param pageSize
     * @param pageNo
     * @return
     * @throws Exception
     */
    @Override
    public RepByPage<NodeModel> getK8sClusterNodeInfosByConditions(String cloudVendor,
                                                       String k8sClusterName, String keyword,
                                                       int pageSize, int pageNo) throws Exception {

        return getK8sClusterNodeInfosByConditions(cloudVendor,k8sClusterName,null,keyword,pageSize,pageNo);
    }

    /**
     *
     * @param nameSpace
     * @param k8sClient
     * @return
     */
    @Override
    public List<PodMetrics> getPodMetricsByNs(String nameSpace, KubernetesClient k8sClient) {

        List<PodMetrics> result = new ArrayList<>();
        if (StringUtils.isEmpty(nameSpace)){
            return result;
        }

        if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,nameSpace)) {
            if ((k8sClient.top() == null) || (k8sClient.top().pods() == null)) {
                return result;
            }
        }

        if ((k8sClient.top().pods().inNamespace(nameSpace) != null)
                &&(k8sClient.top().pods().inNamespace(nameSpace).metrics() != null)){
            return k8sClient.top().pods().inNamespace(nameSpace).metrics().getItems();
        }

        return result;
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @return
     */
    @Override
    public List<String> getAllNameSpaceNames(K8sClusterBasicModel k8sClusterBasicModel) {

        List<String> result = new ArrayList<>();

        if (!StringUtils.equals(k8sClusterBasicModel.getK8sStatus(),K8sStatusEnum.Running.getChineseDescribe())){
            return result;
        }


        NamespaceList namespaceList = k8sClusterBasicModel.getK8sClient().namespaces().list();
        for (Namespace namespace:namespaceList.getItems()) {

            result.add(namespace.getMetadata().getName());
        }

        return result;
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @param nameSpace
     * @return
     * @throws ParseException
     */
    @Override
    public List<PodModel> getAllPodsInfo(K8sClusterBasicModel k8sClusterBasicModel, String nameSpace) throws ParseException {
        return getAllPodsInfo(k8sClusterBasicModel,nameSpace,null);
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @param nameSpace
     * @param podName
     * @return
     * @throws ParseException
     */
    @Override
    public List<PodModel> getAllPodsInfo(K8sClusterBasicModel k8sClusterBasicModel,String nameSpace,String podName) throws ParseException {

        List<PodModel> result = new ArrayList<>();

        if (!StringUtils.equals(k8sClusterBasicModel.getK8sStatus(),K8sStatusEnum.Running.getChineseDescribe())){
            return result;
        }

        //k8sClient获取
        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        List<Pod> pods = new ArrayList<>();
        Map<String,PodMetrics> podMetricsMap = new HashMap<>();
        if (StringUtils.isEmpty(nameSpace)) {

            pods = k8sClient.pods().inAnyNamespace().list().getItems();
            //获取所有的namespace
            List<String> nameSpaceNames = getAllNameSpaceNames(k8sClusterBasicModel);
            //根据ns放入podmetirc
            for (String nameSpaceName:nameSpaceNames){
                List<PodMetrics> podMetricsList = getPodMetricsByNs(nameSpaceName,k8sClient);
                for (PodMetrics podMetrics:podMetricsList){
                    podMetricsMap.put(nameSpaceName+podMetrics.getMetadata().getName(),podMetrics);
                }

            }
        }else {

            //如果podname不为空，获取pod
            if (StringUtils.isNotEmpty(podName)) {

                pods.add(k8sClient.pods().inNamespace(nameSpace).withName(podName).get());

            } else {
                pods = k8sClient.pods().inNamespace(nameSpace).list().getItems();
            }
            //根据ns获取podMetric
            List<PodMetrics> podMetricsList = getPodMetricsByNs(nameSpace,k8sClient);
            for (PodMetrics podMetrics:podMetricsList){
                podMetricsMap.put(nameSpace+podMetrics.getMetadata().getName(),podMetrics);
            }
        }

        for (Pod pod:pods){

            PodModel podModel = new PodModel();

            //构造基本的pod信息
            podModel.setK8sClusterIp(k8sClusterBasicModel.getK8sClusterIp());
            podModel.setK8sClusterLocation(k8sClusterBasicModel.getK8sClusterLocation());
            podModel.setK8sClusterName(k8sClusterBasicModel.getK8sClusterName());
            podModel.setCreateUser(k8sClusterBasicModel.getCreateUser());
            podModel.setCloudVendor(k8sClusterBasicModel.getCloudVendor());

            podModel.setPodName(pod.getMetadata().getName());
            podModel.setNamespace(pod.getMetadata().getNamespace());
            podModel.setNodeName(pod.getSpec().getNodeName());
            podModel.setPodIP(pod.getStatus().getPodIP());
            podModel.setStatus(pod.getStatus().getPhase());
            pod.getMetadata().getCreationTimestamp();

            List<ContainerModel> containerModels = new ArrayList<>();
            //获取容器
            List<ContainerStatus> containerStatuses = pod.getStatus().getContainerStatuses();
            int maxRestartTimes = 0;
            for (ContainerStatus containerStatus:containerStatuses){
                ContainerModel containerModel = new ContainerModel();

                containerModel.setContainerName(containerStatus.getName());
                containerModel.setContainerImage(containerStatus.getImage());
                containerModel.setRestartTimes(containerStatus.getRestartCount());
                containerModel.setContainerId(containerStatus.getContainerID());
                if (containerModel.getRestartTimes() > maxRestartTimes){
                    maxRestartTimes = containerModel.getRestartTimes();
                }

                if(containerStatus.getState().getRunning() != null) {
                    containerModel.setStatus(CnapCloudMgmtConstants.STATUS_RUNNING);
                }else if (containerStatus.getState().getTerminated() != null){
                    containerModel.setStatus(CnapCloudMgmtConstants.STATUS_TERMINATED);
                }else if (containerStatus.getState().getWaiting() != null){
                    containerModel.setStatus(CnapCloudMgmtConstants.STATUS_WAITING);
                }
                
                containerModels.add(containerModel);
            }
            //获取初始化容器
            List<ContainerStatus> initContainerStatuses = pod.getStatus().getInitContainerStatuses();
            for (ContainerStatus containerStatus:initContainerStatuses){
                ContainerModel containerModel = new ContainerModel();

                containerModel.setContainerName(containerStatus.getName());
                containerModel.setContainerImage(containerStatus.getImage());
                containerModel.setRestartTimes(containerStatus.getRestartCount());
                if (containerModel.getRestartTimes() > maxRestartTimes){
                    maxRestartTimes = containerModel.getRestartTimes();
                }

                if(containerStatus.getState().getRunning() != null) {
                    containerModel.setStatus(CnapCloudMgmtConstants.STATUS_RUNNING);
                }else if (containerStatus.getState().getTerminated() != null){
                    containerModel.setStatus(CnapCloudMgmtConstants.STATUS_TERMINATED);
                }else if (containerStatus.getState().getWaiting() != null){
                    containerModel.setStatus(CnapCloudMgmtConstants.STATUS_WAITING);
                }

                containerModels.add(containerModel);
            }

            podModel.setContainerModels(containerModels);
            podModel.setContainerNum(containerModels.size());

            podModel.setRestartTimes(maxRestartTimes);

            podModel.setAge(DateUtils.countTimeBetweenNow(pod.getMetadata().getCreationTimestamp()));

            //获取节点资源使用情况
            PodMetrics podMetrics = podMetricsMap.get(podModel.getNamespace()+podModel.getPodName());
            if (podMetrics != null) {
                List<ContainerMetrics> containerMetricsList = podMetrics.getContainers();
                Long cpuUseByN = 0l;
                long memUseByKi = 0l;
                for (ContainerMetrics containerMetrics : containerMetricsList) {
                    Map<String, Quantity> quantityMap = containerMetrics.getUsage();
                    Quantity cpuQuantity = quantityMap.get("cpu");
                    if (cpuQuantity != null) {
                        cpuUseByN += QuantityUtils.countCpuUseToN(cpuQuantity);
                    }

                    Quantity memQuantity = quantityMap.get("memory");
                    if (memQuantity != null) {
                        memUseByKi += QuantityUtils.countMemUseToKi(memQuantity);
                    }
                }

                podModel.setCpuUseByN(cpuUseByN);
                podModel.setMemUseByKi(memUseByKi);
                podModel.setCpuUse(QuantityUtils.changeCpuNtoM(cpuUseByN));
                podModel.setMemUse(QuantityUtils.changeMemKitoMi(memUseByKi));
            }

            //获取requset的资源要求和limit
            List<Container> containers = pod.getSpec().getContainers();
            Long podReqCpuByN = 0L;
            Long podReqMemByKi = 0L;
            Long podLimitCpuByN = 0L;
            Long podLimitMemByKi = 0L;
            for(Container container:containers) {

                Map<String, Quantity> reqsMap = container.getResources().getRequests();
                if (reqsMap != null) {

                    // 一核= 1000m 1m = 1000*1000n
                    //没有单位表示核
                    Quantity cpuQuantity = reqsMap.get("cpu");
                    if (cpuQuantity != null) {
                        podReqCpuByN += QuantityUtils.countCpuUseToN(cpuQuantity);
                    }

                    //ki和k是1024和1000的关系
                    Quantity memQuantity = reqsMap.get("memory");
                    if (memQuantity != null) {
                        podReqMemByKi += QuantityUtils.countMemUseToKi(memQuantity);
                    }
                }

                Map<String, Quantity> limitMap = container.getResources().getLimits();
                if (limitMap != null) {
                    if (reqsMap != null) {

                        Quantity cpuQuantity = limitMap.get("cpu");
                        if (cpuQuantity != null) {
                            podLimitCpuByN += QuantityUtils.countCpuUseToN(cpuQuantity);
                        }

                        Quantity memQuantity = limitMap.get("memory");
                        if (memQuantity != null) {
                            podLimitMemByKi += QuantityUtils.countMemUseToKi(memQuantity);
                        }
                    }
                }
            }
            podModel.setPodReqCpuByN(podReqCpuByN);
            podModel.setPodReqMemByKi(podReqMemByKi);
            podModel.setPodLimitCpuByN(podLimitCpuByN);
            podModel.setPodLimitMemByKi(podLimitMemByKi);

            //获取他对应的管理者
            List<OwnerReference> ownerReferences = pod.getMetadata().getOwnerReferences();
            WorkLoadModel workLoadModel = getOwnerReference(podModel.getNamespace(),k8sClient,ownerReferences);
            if (workLoadModel != null){
                podModel.setWorkLoadModel(workLoadModel);
            }

            //增加pod的labels
            podModel.setPodLabelMap(pod.getMetadata().getLabels());

            //完成参数查询
            result.add(podModel);

            if (StringUtils.equals(podModel.getPodName(),podName)){
                break;
            }
        }

        return result;
    }

    /**
     *
     * @param nameSpace
     * @param k8sClient
     * @param replicaSetName
     * @return
     */
    @Override
    public ReplicaSet getReplicaSetByName(String nameSpace, KubernetesClient k8sClient, String replicaSetName) {

        if (StringUtils.isEmpty(nameSpace)){
            return null;
        }
        if ((k8sClient.apps().replicaSets().inNamespace(nameSpace) != null)
                &&(k8sClient.apps().replicaSets().inNamespace(nameSpace).withName(replicaSetName) != null)){

            return k8sClient.apps().replicaSets()
                    .inNamespace(nameSpace).withName(replicaSetName).get();
        }

        return null;
    }

    /**
     *
     * @param nameSpace
     * @param k8sClient
     * @param deploymentName
     * @return
     */
    @Override
    public Deployment getDeploymentByName(String nameSpace, KubernetesClient k8sClient, String deploymentName) {

        if ((k8sClient.apps().deployments().inNamespace(nameSpace) != null)
                &&(k8sClient.apps().deployments().inNamespace(nameSpace).withName(deploymentName) != null)) {
            
            return k8sClient.apps().deployments()
                    .inNamespace(nameSpace).withName(deploymentName).get();
        }

        return null;
    }

    /**
     *
     * @param nameSpace
     * @param k8sClient
     * @param statefulSetName
     * @return
     */
    @Override
    public StatefulSet getStatefulSetByName(String nameSpace, KubernetesClient k8sClient, String statefulSetName) {

        if ((k8sClient.apps().statefulSets().inNamespace(nameSpace) != null)
                &&(k8sClient.apps().statefulSets().inNamespace(nameSpace).withName(statefulSetName) != null)){

            return k8sClient.apps().statefulSets().inNamespace(nameSpace).withName(statefulSetName).get();
        }

        return null;
        
    }

    /**
     *
     * @param nameSpace
     * @param k8sClient
     * @param daemonSetName
     * @return
     */
    @Override
    public DaemonSet getDaemonSetByName(String nameSpace, KubernetesClient k8sClient, String daemonSetName) {

        if ((k8sClient.apps().daemonSets().inNamespace(nameSpace) != null)
                &&(k8sClient.apps().daemonSets().inNamespace(nameSpace).withName(daemonSetName) != null)){

            return k8sClient.apps().daemonSets().inNamespace(nameSpace).withName(daemonSetName).get();
        }

        return null;

    }

    /**
     * 获取隶属关系，直到最上层
     * @param nameSpace
     * @param k8sClient
     * @param ownerReferences
     * @return
     */
    @Override
    public WorkLoadModel getOwnerReference(String nameSpace,KubernetesClient k8sClient,List<OwnerReference> ownerReferences) {

        if ((ownerReferences == null)||(ownerReferences.isEmpty())){
            return null;
        }
        if (StringUtils.isEmpty(nameSpace)){
            return null;
        }

        WorkLoadModel workLoadModel = null;
        for (OwnerReference ownerReference:ownerReferences) {

            if (StringUtils.equals(ownerReference.getKind(), CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_REPLICASET)) {
                //获取对应的ReplicaSet
                ReplicaSet replicaSet = getReplicaSetByName(nameSpace, k8sClient, ownerReference.getName());
                if (replicaSet != null) {
                    List<OwnerReference> ownerReferenceList = replicaSet.getMetadata().getOwnerReferences();
                    workLoadModel = getOwnerReference(nameSpace, k8sClient, ownerReferenceList);
                    if (workLoadModel == null) {
                        workLoadModel = new WorkLoadModel();
                        workLoadModel.setWorkLoadName(replicaSet.getMetadata().getName());
                        workLoadModel.setWorkLoadKind(CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_REPLICASET);
                        break;
                    }
                }
            }

            if (StringUtils.equals(ownerReference.getKind(), CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DEPLOYMENT)){

                Deployment deployment = null;
                if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,nameSpace)){
                    deployment = new Deployment();
                    deployment.setMetadata(new ObjectMeta());
                    deployment.getMetadata().setOwnerReferences(new ArrayList<>());
                    deployment.getMetadata().setName(CnapCloudMgmtConstants.TEST_ADMIN);

                }else {
                    deployment = getDeploymentByName(nameSpace, k8sClient, ownerReference.getName());
                }
                if (deployment != null){
                    List<OwnerReference> ownerReferenceList = deployment.getMetadata().getOwnerReferences();
                    workLoadModel = getOwnerReference(nameSpace, k8sClient, ownerReferenceList);
                    if (workLoadModel == null) {
                        workLoadModel = new WorkLoadModel();
                        workLoadModel.setWorkLoadName(deployment.getMetadata().getName());
                        workLoadModel.setWorkLoadKind(CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DEPLOYMENT);
                        break;
                    }
                }
            }


            if (StringUtils.equals(ownerReference.getKind(), CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_STATEFULSET)){

                StatefulSet statefulSet = null;
                if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,nameSpace)){
                    statefulSet = new StatefulSet();
                    statefulSet.setMetadata(new ObjectMeta());
                    statefulSet.getMetadata().setName(CnapCloudMgmtConstants.TEST_ADMIN);
                    statefulSet.getMetadata().setOwnerReferences(new ArrayList<>());

                }else {
                    statefulSet = getStatefulSetByName(nameSpace, k8sClient, ownerReference.getName());
                }
                if (statefulSet != null){
                    List<OwnerReference> ownerReferenceList = statefulSet.getMetadata().getOwnerReferences();
                    workLoadModel = getOwnerReference(nameSpace, k8sClient, ownerReferenceList);
                    if (workLoadModel == null) {
                        workLoadModel = new WorkLoadModel();
                        workLoadModel.setWorkLoadName(statefulSet.getMetadata().getName());
                        workLoadModel.setWorkLoadKind( CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_STATEFULSET);
                        break;
                    }
                }
            }

            if (StringUtils.equals(ownerReference.getKind(), CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DAEMONSET)){

                DaemonSet daemonSet = null;
                if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,nameSpace)) {

                    daemonSet = new DaemonSet();
                    daemonSet.setMetadata(new ObjectMeta());
                    daemonSet.getMetadata().setName(CnapCloudMgmtConstants.TEST_ADMIN);

                }else {
                    daemonSet = getDaemonSetByName(nameSpace, k8sClient, ownerReference.getName());
                }
                if (daemonSet != null){
                    List<OwnerReference> ownerReferenceList = daemonSet.getMetadata().getOwnerReferences();
                    workLoadModel = getOwnerReference(nameSpace, k8sClient, ownerReferenceList);
                    if (workLoadModel == null) {
                        workLoadModel = new WorkLoadModel();
                        workLoadModel.setWorkLoadName(daemonSet.getMetadata().getName());
                        workLoadModel.setWorkLoadKind(CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DAEMONSET);
                        break;
                    }
                }
            }

            if (workLoadModel != null){
                break;
            }
        }

        return workLoadModel;
    }

    /**
     *
     * @param k8sClusterIp
     * @param nodeName
     * @throws Exception
     */
    @Override
    public void delK8sClusterNode(String k8sClusterIp, String nodeName) throws Exception {
        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByIp(k8sClusterIp);

        LOGGER.warn("delK8sClusterNode begin k8sClusterIp={} nodeName={}",k8sClusterIp,nodeName);

        //把安装时要做的删除做掉
        try {
            //找到master和要删除的
            VmModel masterModel = new VmModel();
            VmModel delVmModel = null;
            //更新vmjson
            List<VmModel> newVmModels = new ArrayList<>();

            for (VmModel vmModel:k8sClusterBasicModel.getVmModels()){
                if (StringUtils.equalsIgnoreCase(vmModel.getK8sCluterRole(),
                        CnapCloudMgmtConstants.CNAP_VM_MASTER)){
                    masterModel = vmModel;
                }

                if (StringUtils.equalsIgnoreCase(nodeName,
                        CnapCloudMgmtConstants.CNAP_NODE_PREFIX+vmModel.getInnerIp())){
                    delVmModel = vmModel;
                }else {
                    //新的vmmodel里面加上这个要删除的
                    newVmModels.add(vmModel);
                }
            }

            //如果找不到delModel，或者delModel是master，就不处理
            if ((delVmModel == null)
                    ||(!StringUtils.equalsIgnoreCase(nodeName,
                    CnapCloudMgmtConstants.CNAP_NODE_PREFIX+delVmModel.getInnerIp()))
                    ||(StringUtils.equalsIgnoreCase(delVmModel.getK8sCluterRole(),
                    CnapCloudMgmtConstants.CNAP_VM_MASTER))){
                return;
            }

            if (StringUtils.isNotEmpty(masterModel.getInnerIp())
                &&StringUtils.isNotEmpty(delVmModel.getInnerIp())) {
                //删除文件中的配置
                cnapSshService.delHostMumo(masterModel, delVmModel);
                //删除免密登陆配置
                cnapSshService.delInnerIpNoPasswdSsh(masterModel, delVmModel);
            }


            List<VmModel> toDelVmModels = new ArrayList<>();
            toDelVmModels.add(delVmModel);

            //更新vmJson
            String vmJson = JSONObject.toJSONString(newVmModels);
            multiCloudMgmtService.updateVmJson(k8sClusterBasicModel,vmJson);

            LOGGER.warn("delK8sClusterNode to platform del node name={} id={}",nodeName,delVmModel.getVmId());
            vmResourceService.delVms(toDelVmModels);

        }catch (Exception e){
            LOGGER.error("delK8sClusterNode fail k8sname={} nodename={}",k8sClusterBasicModel.getK8sClusterName(),
                    nodeName,e);
        }

        if (!StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),
                CnapCloudMgmtConstants.TEST_ADMIN)) {

            delK8sClusterNode(k8sClusterBasicModel, nodeName);
        }

    }

    @Override
    public void delK8sClusterNode(K8sClusterBasicModel k8sClusterBasicModel, String nodeName) throws Exception {

        LOGGER.warn("delK8sClusterNode cluster={} nodeName={}",k8sClusterBasicModel.getK8sClusterIp(),nodeName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        List<NodeModel> nodeModels = null;
        //ut
        if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){
            nodeModels = new ArrayList<>();
            NodeModel nodeModel = new NodeModel();
            nodeModels.add(nodeModel);
            nodeModel.setNodeRole(K8sRoleEnum.Slave.getStatus());
        }else {

            nodeModels = getAllNodeInfo(k8sClusterBasicModel, nodeName, null, null);
        }
        if ((nodeModels == null)||(nodeModels.isEmpty())){
            return;
        }
        if (StringUtils.equals(nodeModels.get(0).getNodeRole(),K8sRoleEnum.Master.getStatus())){
            throw new Exception("master 节点不允许删除");
        }

        if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){

            k8sClient.nodes().withName(nodeName).delete();
            //统一记录操作日志
            operationLogService.saveOperationLog(OperationTypeEnum.DELETE.getType(), "删除集群节点：" + nodeName);
        }
    }

    @Override
    public RepByPage<NameSpaceModel> getK8sClusterNameSpaces(String cloudVendor, String k8sClusterName,
                                                             String keyword, int pageSize, int pageNo,
                                                             String accountName) throws Exception {
        List<K8sClusterBasicModel> k8sClusterBasicModels = multiCloudMgmtService
                .getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName,accountName);
        if ((k8sClusterBasicModels == null)||(k8sClusterBasicModels.isEmpty())){
            RepByPage<NameSpaceModel> result = new RepByPage<>();
            result.setPageSize(pageSize);
            result.setPageNo(pageNo);
            result.setTotalNum(0);
            result.setResult(new ArrayList<>());
            return result;
        }

        List<NameSpaceModel> allNameSpaceModels = new ArrayList<>();
        for (K8sClusterBasicModel k8sClusterBasicModel:k8sClusterBasicModels) {
            allNameSpaceModels.addAll(getAllNameSpaceInfo(k8sClusterBasicModel));
        }

        if (StringUtils.isNotEmpty(keyword)){

            List<NameSpaceModel> nameSpaceModelsByKeyWord = new ArrayList<>();
            for (NameSpaceModel nameSpaceModel:allNameSpaceModels){

                if (StringUtils.contains(nameSpaceModel.getNameSpaceName(),keyword)
                        ||(StringUtils.contains(nameSpaceModel.getDescribe(),keyword))){

                    nameSpaceModelsByKeyWord.add(nameSpaceModel);
                }
            }

            return OtherUtils.getResultFromList(nameSpaceModelsByKeyWord,pageNo,pageSize);

        }

        return OtherUtils.getResultFromList(allNameSpaceModels,pageNo,pageSize);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param keyword
     * @param pageSize
     * @param pageNo
     * @return
     * @throws Exception
     */
    @Override
    public RepByPage<NameSpaceModel> getK8sClusterNameSpaces(String cloudVendor, String k8sClusterName,
                                                             String keyword, int pageSize, int pageNo) throws Exception {

        return getK8sClusterNameSpaces(cloudVendor,k8sClusterName,keyword,pageSize,pageNo,null);
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @return
     * @throws ParseException
     */
    @Override
    public List<NameSpaceModel> getAllNameSpaceInfo(K8sClusterBasicModel k8sClusterBasicModel) throws ParseException {

        List<NameSpaceModel> result = new ArrayList<>();
        if (!StringUtils.equals(k8sClusterBasicModel.getK8sStatus(),K8sStatusEnum.Running.getChineseDescribe())){
            return result;
        }

        NamespaceList namespaceList = k8sClusterBasicModel.getK8sClient().namespaces().list();
        for (Namespace namespace:namespaceList.getItems()){

            NameSpaceModel nameSpaceModel = new NameSpaceModel();

            nameSpaceModel.setK8sClusterIp(k8sClusterBasicModel.getK8sClusterIp());
            nameSpaceModel.setCloudVendor(k8sClusterBasicModel.getCloudVendor());
            nameSpaceModel.setK8sClusterLocation(k8sClusterBasicModel.getK8sClusterLocation());
            nameSpaceModel.setK8sClusterName(k8sClusterBasicModel.getK8sClusterName());
            nameSpaceModel.setCreateUser(k8sClusterBasicModel.getCreateUser());

            nameSpaceModel.setNameSpaceName(namespace.getMetadata().getName());

            if (namespace.getMetadata().getAnnotations().containsKey(CnapCloudMgmtConstants.CNAP_DESCRIBE)){
                nameSpaceModel.setDescribe(
                        namespace.getMetadata().getAnnotations().get(CnapCloudMgmtConstants.CNAP_DESCRIBE));
            }

            List<PodModel> podModels = getAllPodsInfo(k8sClusterBasicModel,nameSpaceModel.getNameSpaceName());
            nameSpaceModel.setPodNum(podModels.size());

            //cpu使用中
            long cpuUse = 0L;
            long memUse = 0L;
            Map<String,String> workLoadMap = new HashMap<>();
            for(PodModel podModel :podModels){
                cpuUse += podModel.getCpuUseByN();
                memUse += podModel.getMemUseByKi();
                if (podModel.getWorkLoadModel() != null) {
                    workLoadMap.put(podModel.getWorkLoadModel().getWorkLoadName(),
                            podModel.getWorkLoadModel().getWorkLoadKind());
                }
            }
            nameSpaceModel.setCpuUse(QuantityUtils.changeCpuNtoM(cpuUse));
            nameSpaceModel.setMemUse(QuantityUtils.changeMemKitoMi(memUse));
            nameSpaceModel.setDeploymentNum(workLoadMap.size());

            List<SvcModel> svcModels = getAllSvcInfo(k8sClusterBasicModel,nameSpaceModel.getNameSpaceName());
            nameSpaceModel.setSvcNum(svcModels.size());

            nameSpaceModel.setCreateTime(
                    DateUtils.getDateByYYMMDDTHHMMSS(namespace.getMetadata().getCreationTimestamp()));

            result.add(nameSpaceModel);
        }

        result.sort((t1, t2) -> (int)(t1.getCreateTime().getTime() - t2.getCreateTime().getTime()));

        return result;
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @return
     * @throws Exception
     */
    @Override
    public int getAllSvcNum(String cloudVendor, String k8sClusterName) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel = multiCloudMgmtService
                .getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        return k8sClusterBasicModel.getK8sClient().services().inAnyNamespace().list().getItems().size();
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @param nameSpace
     * @param svcName
     * @return
     * @throws ParseException
     */
    @Override
    public SvcModel getSvcInfo(K8sClusterBasicModel k8sClusterBasicModel, String nameSpace, String svcName) throws ParseException {

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        io.fabric8.kubernetes.api.model.Service svc = null;
        if (StringUtils.isEmpty(nameSpace)){
            List<io.fabric8.kubernetes.api.model.Service>  serviceList =
                    k8sClient.services().inAnyNamespace().list().getItems();
            for (io.fabric8.kubernetes.api.model.Service tmpSvc:serviceList){

                if ((tmpSvc == null)||(tmpSvc.getMetadata() == null)){
                    continue;
                }

                if (StringUtils.equalsIgnoreCase(tmpSvc.getMetadata().getName(),svcName)){
                    svc = tmpSvc;
                    break;
                }
            }


        }else {
            svc =  k8sClient.services().inNamespace(nameSpace).withName(svcName).get();
        }

        if ((svc == null)||(svc.getMetadata() == null)){
            return null;
        }

        SvcModel svcModel = new SvcModel();

        svcModel.setK8sClusterIp(k8sClusterBasicModel.getK8sClusterIp());
        svcModel.setK8sClusterName(k8sClusterBasicModel.getK8sClusterName());
        svcModel.setK8sClusterLocation(k8sClusterBasicModel.getK8sClusterLocation());
        svcModel.setCloudVendor(k8sClusterBasicModel.getCloudVendor());
        svcModel.setCreateUser(k8sClusterBasicModel.getCreateUser());

        svcModel.setSvcName(svc.getMetadata().getName());

        svcModel.setNamespace(svc.getMetadata().getNamespace());

        svcModel.setSvcType(svc.getSpec().getType());

        svcModel.setClusterIp(svc.getSpec().getClusterIP());

        svcModel.setServicePorts(svc.getSpec().getPorts());
        if ((svcModel.getServicePorts() != null)&&(!svcModel.getServicePorts().isEmpty())) {

            StringBuilder portInfoSb = new StringBuilder();

            for (ServicePort servicePort : svc.getSpec().getPorts()) {

                    if (servicePort.getPort() != null) {
                        portInfoSb.append(servicePort.getPort());
                    }

                    if (servicePort.getNodePort() != null) {
                        portInfoSb.append(":").append(servicePort.getNodePort());
                    }

                    if (servicePort.getProtocol() != null) {
                        portInfoSb.append("/").append(servicePort.getProtocol());
                    }

                    portInfoSb.append(" ");
                }
                svcModel.setPort(portInfoSb.toString());
        }

        Map<String, String> selectorMap = svc.getSpec().getSelector();
        StringBuilder selectorSb = new StringBuilder();
        List<KeyValueModel> seclectorList = new ArrayList<>();
        if (selectorMap != null){
                for (String key:selectorMap.keySet()){
                    selectorSb.append(key).append("|").append(selectorMap.get(key));
                    seclectorList.add(new KeyValueModel(key,selectorMap.get(key)));
                }
        }
        svcModel.setSeclectorList(seclectorList);
        svcModel.setSeclector(selectorSb.toString());
        svcModel.setSeclectorMap(selectorMap);

        svcModel.setStatus("Active");

        svcModel.setCreateTime(
                    DateUtils.getDateByYYMMDDTHHMMSS(svc.getMetadata().getCreationTimestamp()));

        svcModel.setYaml(Serialization.asYaml(svc));

        return svcModel;
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @param nameSpace
     * @return
     * @throws ParseException
     */
    @Override
    public List<SvcModel> getAllSvcInfo(K8sClusterBasicModel k8sClusterBasicModel, String nameSpace) throws ParseException {

        List<SvcModel> svcModels = new ArrayList<>();

        if (!StringUtils.equals(k8sClusterBasicModel.getK8sStatus(),K8sStatusEnum.Running.getChineseDescribe())){
            return svcModels;
        }

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        List<io.fabric8.kubernetes.api.model.Service> services = new ArrayList<>();
        if (StringUtils.isEmpty(nameSpace)) {
            services =
                    k8sClient.services().inAnyNamespace().list().getItems();
        }else {

            if (k8sClient.services().inNamespace(nameSpace) != null) {
                services =
                        k8sClient.services().inNamespace(nameSpace).list().getItems();
            }
        }

        for (io.fabric8.kubernetes.api.model.Service svc:services){

            SvcModel svcModel = new SvcModel();

            svcModel.setK8sClusterIp(k8sClusterBasicModel.getK8sClusterIp());
            svcModel.setK8sClusterName(k8sClusterBasicModel.getK8sClusterName());
            svcModel.setK8sClusterLocation(k8sClusterBasicModel.getK8sClusterLocation());
            svcModel.setCloudVendor(k8sClusterBasicModel.getCloudVendor());
            svcModel.setCreateUser(k8sClusterBasicModel.getCreateUser());
            svcModel.setRegionId(k8sClusterBasicModel.getRegionId());
            svcModel.setZoneId(k8sClusterBasicModel.getZoneId());

            svcModel.setSvcName(svc.getMetadata().getName());

            svcModel.setNamespace(svc.getMetadata().getNamespace());

            svcModel.setSvcType(svc.getSpec().getType());

            svcModel.setClusterIp(svc.getSpec().getClusterIP());

            svcModel.setServicePorts(svc.getSpec().getPorts());
            if ((svcModel.getServicePorts() != null)&&(!svcModel.getServicePorts().isEmpty())) {

                    StringBuilder portInfoSb = new StringBuilder();

                    for (ServicePort servicePort : svc.getSpec().getPorts()) {

                        if (servicePort.getPort() != null) {
                            portInfoSb.append(servicePort.getPort());
                        }

                        if (servicePort.getNodePort() != null) {
                            portInfoSb.append(":").append(servicePort.getNodePort());
                        }

                        if (servicePort.getProtocol() != null) {
                            portInfoSb.append("/").append(servicePort.getProtocol());
                        }

                        portInfoSb.append(" ");
                    }
                    svcModel.setPort(portInfoSb.toString());
            }

            Map<String, String> selectorMap = svc.getSpec().getSelector();
            StringBuilder selectorSb = new StringBuilder();
            List<KeyValueModel> seclectorList = new ArrayList<>();
            if (selectorMap != null){
                for (String key:selectorMap.keySet()){
                    selectorSb.append(key).append("|").append(selectorMap.get(key));
                    seclectorList.add(new KeyValueModel(key,selectorMap.get(key)));
                }
            }
            svcModel.setSeclectorList(seclectorList);
            svcModel.setSeclector(selectorSb.toString());
            svcModel.setSeclectorMap(selectorMap);

            svcModel.setStatus("Active");

            svcModel.setCreateTime(
                    DateUtils.getDateByYYMMDDTHHMMSS(svc.getMetadata().getCreationTimestamp()));

            svcModel.setYaml(Serialization.asYaml(svc));

            //处理和微服务的对应关系
            MicroServicePO microServicePO =
                    cacheService.getMicroSvc(svcModel.getCreateUser(),svcModel.getSvcName());
            if (microServicePO != null){
                svcModel.setMicroServiceName(microServicePO.getMicroServiceName());
                svcModel.setMicroServicePO(microServicePO);
                svcModel.setPath(microServicePO.getPath());

                //去寻找这个svc的场景
                String busScence = microSvcMapper.getBusinessScenceBySvcId(microServicePO.getId());
                if (StringUtils.isNotEmpty(busScence)) {
                    SvcBusinessScene svcBusinessScene = JSON.parseObject(busScence, SvcBusinessScene.class);
                    svcModel.setBusinessSceneSvcType(svcBusinessScene.getSvcType());
                }



                //另一个维度的micro的缓存
                cacheService.putSvcModelForMicroSvc(svcModel.getK8sClusterIp() + svcModel.getSvcName(),svcModel);
            }

            svcModels.add(svcModel);

        }

        return svcModels;
    }

    /**
     * 
     * @param nameSpaceModel
     * @throws Exception
     */
    @Override
    public void addK8sClusterNameSpaces(NameSpaceModel nameSpaceModel) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(
                        nameSpaceModel.getCloudVendor(),nameSpaceModel.getK8sClusterName());

        List<String> nsNames = getAllNameSpaceNames(k8sClusterBasicModel);
        for (String names :nsNames){
            if (StringUtils.equals(names,nameSpaceModel.getNameSpaceName())){
                throw new Exception("命名空间已存在");

            }
        }

        Namespace ns = new Namespace();

        ObjectMeta metaData = new ObjectMeta();

        metaData.setName(nameSpaceModel.getNameSpaceName());

        if (StringUtils.isNotEmpty(nameSpaceModel.getDescribe())){

            Map<String,String> annotationsMap = metaData.getAnnotations();
            if (annotationsMap == null){
                annotationsMap = new HashMap<>();
            }
            annotationsMap.put(CnapCloudMgmtConstants.CNAP_DESCRIBE,nameSpaceModel.getDescribe());
            metaData.setAnnotations(annotationsMap);
        }

        ns.setMetadata(metaData);
        
        k8sClusterBasicModel.getK8sClient().namespaces().resource(ns).createOrReplace();

    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param nameSpaceName
     * @throws Exception
     */
    @Override
    public void delK8sClusterNameSpaces(String cloudVendor, String k8sClusterName, String nameSpaceName) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();
        k8sClient.namespaces().withName(nameSpaceName).delete();
    }

    @Override
    public RepByPage<WorkLoadModel> getK8sClusterWorkLoads(String cloudVendor, String k8sClusterName, String keyword, String svcName, int pageSize, int pageNo, String accountName) throws Exception {
        List<K8sClusterBasicModel> k8sClusterBasicModels = multiCloudMgmtService
                .getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName,accountName);
        if ((k8sClusterBasicModels == null)||(k8sClusterBasicModels.isEmpty())){
            RepByPage<WorkLoadModel> result = new RepByPage<>();
            result.setResult(new ArrayList<>());
            result.setPageNo(pageNo);
            result.setTotalNum(0);
            result.setPageSize(pageSize);
            return result;
        }

        List<WorkLoadModel> allWorkLoadsModels = new ArrayList<>();

        for (K8sClusterBasicModel k8sClusterBasicModel:k8sClusterBasicModels) {
            allWorkLoadsModels.addAll(getAllWorkLoads(k8sClusterBasicModel, null));
        }



        if (StringUtils.isNotEmpty(keyword)){

            List<WorkLoadModel> workLoadModelsByKeyWords = new ArrayList<>();
            for (WorkLoadModel workLoadModel:allWorkLoadsModels){

                if (StringUtils.contains(workLoadModel.getWorkLoadName(),keyword)
                        ||(StringUtils.contains(workLoadModel.getDescribe(),keyword))
                        ||StringUtils.contains(workLoadModel.getNamespace(),keyword)
                        ||StringUtils.contains(workLoadModel.getStatus(),keyword)){

                    workLoadModelsByKeyWords.add(workLoadModel);
                }
            }

            return OtherUtils.getResultFromList(workLoadModelsByKeyWords,pageNo,pageSize);

        }

        return OtherUtils.getResultFromList(allWorkLoadsModels,pageNo,pageSize);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param keyword
     * @param svcName
     * @param pageSize
     * @param pageNo
     * @return
     * @throws Exception
     */
    @Override
    public RepByPage<WorkLoadModel> getK8sClusterWorkLoads(String cloudVendor, String k8sClusterName, String keyword, String svcName, int pageSize, int pageNo) throws Exception {

       return getK8sClusterWorkLoads(cloudVendor,k8sClusterName,keyword,svcName,pageSize,pageNo,null);
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @param nameSpace
     * @return
     * @throws ParseException
     */
    @Override
    public List<WorkLoadModel> getAllWorkLoads(K8sClusterBasicModel k8sClusterBasicModel, String nameSpace) throws ParseException {

        List<WorkLoadModel> result = new ArrayList<>();

        if (!StringUtils.equals(k8sClusterBasicModel.getK8sStatus(),K8sStatusEnum.Running.getChineseDescribe())){
            return result;
        }

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        //获取所有的svc情况，这样可以找到svc对应的workload
        List<SvcModel> svcModels = getAllSvcInfo(k8sClusterBasicModel,nameSpace);
        Map<String,List<String>> svcSelectorMap = new HashMap<>();
        for (SvcModel svcModel:svcModels){
            if((svcModel.getSeclectorMap()!= null)
                &&(!svcModel.getSeclectorMap().isEmpty())){

                for (String selectorKey:svcModel.getSeclectorMap().keySet()){
                    String seletor = selectorKey+"|" + svcModel.getSeclectorMap().get(selectorKey);
                    if (!svcSelectorMap.containsKey(seletor)){
                        svcSelectorMap.put(seletor,new ArrayList<>());
                    }
                    svcSelectorMap.get(seletor).add(svcModel.getSvcName());
                }
            }
        }

        List<Deployment> deployments = new ArrayList<>();
        List<StatefulSet> statefulSets = new ArrayList<>();
        if (StringUtils.isEmpty(nameSpace)) {

            if (k8sClient.apps().deployments().inAnyNamespace() != null) {
                deployments = k8sClient.apps().deployments().inAnyNamespace().list().getItems();
            }

            if (k8sClient.apps().statefulSets().inAnyNamespace() != null) {
                statefulSets = k8sClient.apps().statefulSets().inAnyNamespace().list().getItems();
            }

        }else {

            if (k8sClient.apps().deployments().inNamespace(nameSpace) != null){
                deployments = k8sClient.apps().deployments()
                                    .inNamespace(nameSpace).list().getItems();
            }

            if (k8sClient.apps().statefulSets().inNamespace(nameSpace) != null){
                statefulSets = k8sClient.apps().statefulSets()
                                .inNamespace(nameSpace).list().getItems();
            }
            
        }

        for (Deployment deployment : deployments){

            WorkLoadModel workLoadModel = new WorkLoadModel();

            workLoadModel.setK8sClusterIp(k8sClusterBasicModel.getK8sClusterIp());
            workLoadModel.setK8sClusterLocation(k8sClusterBasicModel.getK8sClusterLocation());
            workLoadModel.setK8sClusterName(k8sClusterBasicModel.getK8sClusterName());
            workLoadModel.setCloudVendor(k8sClusterBasicModel.getCloudVendor());
            workLoadModel.setCreateUser(k8sClusterBasicModel.getCreateUser());

            workLoadModel.setWorkLoadName(deployment.getMetadata().getName());

            workLoadModel.setWorkLoadKind(CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DEPLOYMENT);

            //状态
            if (deployment.getStatus().getConditions() != null){

                List<DeploymentCondition> deploymentConditions = deployment.getStatus().getConditions();
                for (DeploymentCondition deploymentCondition:deploymentConditions){
                    if (StringUtils.equals(deploymentCondition.getType(),"Available")){

                        if(StringUtils.equals(deploymentCondition.getStatus(),"True")){

                            workLoadModel.setStatus(K8sStatusEnum.Running.getChineseDescribe());
                        }else {
                            workLoadModel.setStatus(K8sStatusEnum.Error.getChineseDescribe());
                        }
                        break;
                    }

                    if (StringUtils.equals(deploymentCondition.getType(),"Progressing")){

                        if(StringUtils.equals(deploymentCondition.getStatus(),"True")){

                            workLoadModel.setStatus(K8sStatusEnum.Running.getChineseDescribe());
                        }else {
                            workLoadModel.setStatus(K8sStatusEnum.Error.getChineseDescribe());
                        }
                        break;
                    }
                }
            }

            //namespace
            workLoadModel.setNamespace(deployment.getMetadata().getNamespace());

            workLoadModel.setReplicas(OtherUtils.changeIntegerToInt(deployment.getStatus().getReplicas()));
            workLoadModel.setPodNum(OtherUtils.changeIntegerToInt(deployment.getStatus().getAvailableReplicas()));

            workLoadModel.setCreateTime(DateUtils.getDateByYYMMDDTHHMMSS(deployment.getMetadata().getCreationTimestamp()));

            List<Container> containers = deployment.getSpec().getTemplate().getSpec().getContainers();
            List<ContainerModel> containerModels = new ArrayList<>();
            if (containers != null){

                for (Container container:containers){
                    ContainerModel containerModel = new ContainerModel();
                    containerModel.setContainerName(container.getName());
                    containerModel.setContainerImage(container.getImage());
                    containerModels.add(containerModel);
                }
            }
            workLoadModel.setContainerModels(containerModels);

            //设置svc
            List<String> svcNames = new ArrayList<>();
            Map<String, String> labels = deployment.getSpec().getTemplate().getMetadata().getLabels();
            if ((labels != null)&&(!labels.isEmpty())){
                for (String key:labels.keySet()){
                    if (svcSelectorMap.containsKey(key + "|" + labels.get(key))){
                        svcNames.addAll(svcSelectorMap.get(key + "|" + labels.get(key)));
                    }
                }
            }
            StringBuilder sb = new StringBuilder();
            if (!svcNames.isEmpty()){
                for (String svcName: svcNames){
                    if (StringUtils.isNotEmpty(sb.toString())){
                        sb.append(",");
                    }

                    sb.append(svcName);
                }
            }
            workLoadModel.setSvcName(sb.toString());

            //获取yaml
            workLoadModel.setYaml(Serialization.asYaml(deployment));

            result.add(workLoadModel);
        }

        for (StatefulSet statefulSet:statefulSets){

            WorkLoadModel workLoadModel = new WorkLoadModel();

            workLoadModel.setK8sClusterIp(k8sClusterBasicModel.getK8sClusterIp());
            workLoadModel.setK8sClusterLocation(k8sClusterBasicModel.getK8sClusterLocation());
            workLoadModel.setK8sClusterName(k8sClusterBasicModel.getK8sClusterName());
            workLoadModel.setCloudVendor(k8sClusterBasicModel.getCloudVendor());
            workLoadModel.setCreateUser(k8sClusterBasicModel.getCreateUser());

            workLoadModel.setWorkLoadName(statefulSet.getMetadata().getName());

            workLoadModel.setWorkLoadKind(CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_STATEFULSET);
            workLoadModel.setNamespace(statefulSet.getMetadata().getNamespace());

            workLoadModel.setReplicas(statefulSet.getStatus().getReplicas());
            workLoadModel.setPodNum(statefulSet.getStatus().getCurrentReplicas());

            if ((workLoadModel.getReplicas() > 0)
                    &&(workLoadModel.getReplicas() == workLoadModel.getPodNum())){
                workLoadModel.setStatus(K8sStatusEnum.Running.getChineseDescribe());
            }

            List<Container> containers = statefulSet.getSpec().getTemplate().getSpec().getContainers();
            List<ContainerModel> containerModels = new ArrayList<>();
            if (containers != null){

                for (Container container:containers){
                    ContainerModel containerModel = new ContainerModel();
                    containerModel.setContainerName(container.getName());
                    containerModel.setContainerImage(container.getImage());
                    containerModels.add(containerModel);
                }
            }
            workLoadModel.setContainerModels(containerModels);

            workLoadModel.setCreateTime(DateUtils.getDateByYYMMDDTHHMMSS(statefulSet
                                        .getMetadata().getCreationTimestamp()));

            //设置svc
            List<String> svcNames = new ArrayList<>();
            Map<String, String> labels = statefulSet.getSpec().getTemplate().getMetadata().getLabels();
            if ((labels != null)&&(!labels.isEmpty())){
                for (String key:labels.keySet()){
                    if (svcSelectorMap.containsKey(key + "|" + labels.get(key))){
                        svcNames.addAll(svcSelectorMap.get(key + "|" + labels.get(key)));
                    }
                }
            }
            StringBuilder sb = new StringBuilder();
            if (!svcNames.isEmpty()){
                for (String svcName: svcNames){
                    if (StringUtils.isNotEmpty(sb.toString())){
                        sb.append(",");
                    }

                    sb.append(svcName);
                }
            }
            workLoadModel.setSvcName(sb.toString());

            //获取yaml
            workLoadModel.setYaml(Serialization.asYaml(statefulSet));

            result.add(workLoadModel);
            
        }

        result.sort((workLoadModel, t1) -> (int)(workLoadModel.getCreateTime().getTime() - t1.getCreateTime().getTime()));

        return result;
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param yaml
     * @throws Exception
     */
    @Override
    public void editK8sClusterWorkLoadByYaml(String cloudVendor, String k8sClusterName, String yaml) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        KubernetesResource kubernetesResource = null;
        if (StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),
                CnapCloudMgmtConstants.TEST_ADMIN)){
            if (StringUtils.isEmpty(yaml)) {
                Deployment testDeployment = new Deployment();
                testDeployment.setMetadata(new ObjectMeta());
                kubernetesResource = testDeployment;
            }else{
                StatefulSet testStatefulSet = new StatefulSet();
                testStatefulSet.setMetadata(new ObjectMeta());
                kubernetesResource = testStatefulSet;
            }

        }else {
            kubernetesResource = Serialization.unmarshal(yaml);
        }

        if (kubernetesResource instanceof Deployment){
            Deployment newDeployment = (Deployment) kubernetesResource;
            String namespace = newDeployment.getMetadata().getNamespace();
            if (StringUtils.isEmpty(namespace)){
                newDeployment.getMetadata().setNamespace(CnapCloudMgmtConstants.DEFAULT_NAMESPACE);
            }

            if (StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),
                    CnapCloudMgmtConstants.TEST_ADMIN)){
                return;
            }

            if(k8sClient.apps().deployments().inNamespace(newDeployment.getMetadata().getNamespace())
                    .withName(newDeployment.getMetadata().getName()).get() == null){
                throw new Exception("该工作负载不存在");
            }

            k8sClient.apps().deployments().inNamespace(newDeployment.getMetadata().getNamespace())
                    .resource((Deployment) kubernetesResource).replace();
        }

        if (kubernetesResource instanceof StatefulSet){
            StatefulSet newStatefulSet = (StatefulSet)kubernetesResource;
            if (StringUtils.isEmpty(newStatefulSet.getMetadata().getNamespace())){
                newStatefulSet.getMetadata().setNamespace(CnapCloudMgmtConstants.DEFAULT_NAMESPACE);
            }

            if (StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),
                    CnapCloudMgmtConstants.TEST_ADMIN)){
                return;
            }
            k8sClient.apps().statefulSets().inNamespace(newStatefulSet.getMetadata().getNamespace())
                    .resource((StatefulSet)kubernetesResource).createOrReplace();
        }
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param yaml
     * @throws Exception
     */
    @Override
    public void addK8sClusterWorkLoad(String cloudVendor, String k8sClusterName, String yaml) throws Exception {
        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

       KubernetesResource kubernetesResource = Serialization.unmarshal(yaml);

        if (kubernetesResource instanceof Deployment){
            Deployment newDeployment = (Deployment) kubernetesResource;
            String namespace = newDeployment.getMetadata().getNamespace();
            if (StringUtils.isEmpty(namespace)){
                newDeployment.getMetadata().setNamespace(CnapCloudMgmtConstants.DEFAULT_NAMESPACE);
            }

            if(k8sClient.apps().deployments().inNamespace(newDeployment.getMetadata().getNamespace())
                                            .withName(newDeployment.getMetadata().getName()).get() != null){
                throw new Exception("该工作负载已存在");
            }

            if (k8sClient.apps().deployments().inNamespace(newDeployment.getMetadata().getNamespace())
                    .resource((Deployment) kubernetesResource) != null) {
                k8sClient.apps().deployments().inNamespace(newDeployment.getMetadata().getNamespace())
                        .resource((Deployment) kubernetesResource).createOrReplace();
            }
        }

        if (kubernetesResource instanceof StatefulSet){
            StatefulSet newStatefulSet = (StatefulSet)kubernetesResource;
            if (StringUtils.isEmpty(newStatefulSet.getMetadata().getNamespace())){
                newStatefulSet.getMetadata().setNamespace(CnapCloudMgmtConstants.DEFAULT_NAMESPACE);
            }
            k8sClient.apps().statefulSets().inNamespace(newStatefulSet.getMetadata().getNamespace())
                    .resource((StatefulSet)kubernetesResource).createOrReplace();
        }
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param namespace
     * @param workdLoadName
     * @param workLoadKind
     * @param replicas
     * @throws Exception
     */
    @Override
    public void editK8sClusterWorkLoadReplicas(String cloudVendor, String k8sClusterName,String namespace, String workdLoadName,
                                               String workLoadKind, int replicas) throws Exception {
        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        if (StringUtils.equals(workLoadKind,CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DEPLOYMENT)){
            k8sClient.apps().deployments().inNamespace(namespace).withName(workdLoadName).scale(replicas);
        }else if (StringUtils.equals(workLoadKind,CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_STATEFULSET)){
            k8sClient.apps().statefulSets().inNamespace(namespace).withName(workdLoadName).scale(replicas);
        }
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param workLoadName
     * @param namespace
     * @param workLoadKind
     * @param containerModels
     * @throws Exception
     */
    @Override
    public void editK8sClusterWorkLoadContainers(String cloudVendor, String k8sClusterName,
                                                 String workLoadName, String namespace, String workLoadKind,
                                                 List<ContainerModel> containerModels) throws Exception {
        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();
        Map<String,ContainerModel> containerModelMap = new HashMap<>();
        for (ContainerModel containerModel:containerModels){
            containerModelMap.put(containerModel.getContainerName(),containerModel);
        }

        if (StringUtils.equals(workLoadKind,CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DEPLOYMENT)){

            /**
             * ut
             * 检查能否正常增加deployemnt
             * 其中主要是各个container
             */
            Deployment deployment = null;
            if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){

                deployment = new Deployment();
                deployment.setSpec(new DeploymentSpec());
                deployment.getSpec().setTemplate(new PodTemplateSpec());
                deployment.getSpec().getTemplate().setSpec(new PodSpec());
                List<Container> oldContainers = new ArrayList<>();
                Container container = new Container();
                oldContainers.add(container);
                container.setName("containerName");
                deployment.getSpec().getTemplate().getSpec().setContainers(oldContainers);

            }else{
                deployment = k8sClient.apps().deployments().inNamespace(namespace)
                        .withName(workLoadName).get();
            }


            List<Container> oldContainers = deployment.getSpec().getTemplate().getSpec().getContainers();
            List<Container> newContainers = new ArrayList<>();
            for (Container oldContainer: oldContainers){

                if (containerModelMap.containsKey(oldContainer.getName())){
                    oldContainer.setImage(containerModelMap.get(oldContainer.getName()).getContainerImage());
                }

                newContainers.add(oldContainer);
            }

            deployment.getSpec().getTemplate().getSpec().setContainers(newContainers);
            if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {
                k8sClient.apps().deployments().inNamespace(namespace).resource(deployment).createOrReplace();
            }

        }else if (StringUtils.equals(workLoadKind,CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_STATEFULSET)){

            StatefulSet statefulSet = null;
            if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){
                statefulSet = new StatefulSet();
                statefulSet.setSpec(new StatefulSetSpec());
                statefulSet.getSpec().setTemplate(new PodTemplateSpec());
                statefulSet.getSpec().getTemplate().setSpec(new PodSpec());
                List<Container> oldContainers = new ArrayList<>();
                Container container = new Container();
                oldContainers.add(container);
                container.setName("containerName");
                statefulSet.getSpec().getTemplate().getSpec().setContainers(oldContainers);
            }else {

                statefulSet = k8sClient.apps().statefulSets().inNamespace(namespace)
                        .withName(workLoadName).get();
            }

            List<Container> oldContainers = statefulSet.getSpec().getTemplate().getSpec().getContainers();
            List<Container> newContainers = new ArrayList<>();
            for (Container oldContainer: oldContainers){

                if (containerModelMap.containsKey(oldContainer.getName())){
                    oldContainer.setImage(containerModelMap.get(oldContainer.getName()).getContainerImage());
                }

                newContainers.add(oldContainer);
            }

            statefulSet.getSpec().getTemplate().getSpec().setContainers(newContainers);
            if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {
                k8sClient.apps().statefulSets().inNamespace(namespace).resource(statefulSet).createOrReplace();
            }
        }
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param namespace
     * @param workLoadName
     * @param workLoadKind
     * @throws Exception
     */
    @Override
    public void delK8sClusterWorkLoad(String cloudVendor, String k8sClusterName, String namespace, String workLoadName, String workLoadKind) throws Exception {
        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        if (StringUtils.equals(workLoadKind,CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DEPLOYMENT)){
            k8sClient.apps().deployments().inNamespace(namespace).withName(workLoadName).delete();
        }else if (StringUtils.equals(workLoadKind,CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_STATEFULSET)){
            k8sClient.apps().statefulSets().inNamespace(namespace).withName(workLoadName).delete();
        }
    }


    @Override
    public RepByPage<PodModel> getK8sClusterPods(String cloudVendor, String k8sClusterName, String keyword, String namespace, String svcName, String workLoadName, String nodeName, int pageSize, int pageNo, String accountName) throws Exception {


        //获取基本的操作对象
        List<K8sClusterBasicModel> k8sClusterBasicModels
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName,accountName);
        if ((k8sClusterBasicModels == null)||(k8sClusterBasicModels.isEmpty())){
            RepByPage<PodModel> result = new RepByPage<>();
            result.setPageSize(pageSize);
            result.setPageNo(pageNo);
            result.setTotalNum(0);
            result.setResult(new ArrayList<>());
            return result;
        }

        //获取所有pod，不分条件
        List<PodModel> allPodModels =  new ArrayList<>();
        Map<String,K8sClusterBasicModel> k8sClusterBasicModelMap = new HashMap<>();
        for (K8sClusterBasicModel k8sClusterBasicModel:k8sClusterBasicModels) {
            allPodModels.addAll(getAllPodsInfo(k8sClusterBasicModel, namespace));
            k8sClusterBasicModelMap.put(k8sClusterBasicModel.getK8sClusterName(),k8sClusterBasicModel);
        }


        List<PodModel> podModelsByKeyWords = new ArrayList<>();
        if (StringUtils.isNotEmpty(nodeName)){

            for (PodModel podModel:allPodModels){

                if (StringUtils.equals(podModel.getNodeName(),nodeName)){

                    if (StringUtils.isNotEmpty(keyword)){

                        if (StringUtils.contains(podModel.getPodName(),keyword)
                                ||StringUtils.contains(podModel.getNamespace(),keyword)
                                ||StringUtils.contains(podModel.getNodeName(),keyword)
                                ||StringUtils.contains(podModel.getPodIP(),keyword)
                                ||StringUtils.contains(podModel.getStatus(),keyword)){

                            podModelsByKeyWords.add(podModel);
                        }

                    }else {

                        podModelsByKeyWords.add(podModel);
                    }
                }

            }

            return OtherUtils.getResultFromList(podModelsByKeyWords,pageNo,pageSize);
        }

        if (StringUtils.isNotEmpty(keyword)){

            for (PodModel podModel:allPodModels){

                if (StringUtils.contains(podModel.getPodName(),keyword)
                        ||StringUtils.contains(podModel.getNamespace(),keyword)
                        ||StringUtils.contains(podModel.getNodeName(),keyword)
                        ||StringUtils.contains(podModel.getPodIP(),keyword)
                        ||StringUtils.contains(podModel.getStatus(),keyword)){
                    podModelsByKeyWords.add(podModel);
                }

            }
            return OtherUtils.getResultFromList(podModelsByKeyWords,pageNo,pageSize);
        }

        if (StringUtils.isNotEmpty(workLoadName)){

            for (PodModel podModel:allPodModels){

                if ((podModel.getWorkLoadModel() != null)
                        &&(StringUtils.equals(podModel.getWorkLoadModel().getWorkLoadName(),workLoadName))){
                    podModelsByKeyWords.add(podModel);
                }

            }
            return OtherUtils.getResultFromList(podModelsByKeyWords,pageNo,pageSize);
        }

        if ((StringUtils.isNotEmpty(svcName)
                &&StringUtils.isNotEmpty(k8sClusterName))){

            //获取svc的selectorLabel
            SvcModel svcModel = getSvcModelByName(k8sClusterBasicModelMap.get(k8sClusterName), namespace, svcName);

            if (StringUtils.isEmpty(svcModel.getSvcName())){
                return OtherUtils.getResultFromList(podModelsByKeyWords,pageNo,pageSize);
            }

            for (PodModel podModel:allPodModels){

                if ((podModel.getPodLabelMap() != null)&&(!podModel.getPodLabelMap().isEmpty())){

                    for (String podLabelKey: podModel.getPodLabelMap().keySet()){
                        if (svcModel.getSeclectorMap().containsKey(podLabelKey)
                                &&(StringUtils.equals(
                                podModel.getPodLabelMap().get(podLabelKey),
                                svcModel.getSeclectorMap().get(podLabelKey)))){
                            podModelsByKeyWords.add(podModel);
                            break;
                        }
                    }
                }
            }
            return OtherUtils.getResultFromList(podModelsByKeyWords,pageNo,pageSize);
        }

        return OtherUtils.getResultFromList(allPodModels,pageNo,pageSize);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param keyword
     * @param namespace
     * @param svcName
     * @param workLoadName
     * @param nodeName
     * @param pageSize
     * @param pageNo
     * @return
     * @throws Exception
     */
    @Override
    public RepByPage<PodModel> getK8sClusterPods(String cloudVendor, String k8sClusterName,
                                                 String keyword, String namespace, String svcName,
                                                 String workLoadName, String nodeName,
                                                 int pageSize, int pageNo) throws Exception {
        return getK8sClusterPods(cloudVendor,k8sClusterName,
                keyword, namespace, svcName,
                workLoadName, nodeName, pageSize, pageNo,null);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param keyword
     * @param namespace
     * @param svcName
     * @param workLoadName
     * @param pageSize
     * @param pageNo
     * @return
     * @throws Exception
     */
    @Override
    public RepByPage<PodModel> getK8sClusterPods(String cloudVendor, String k8sClusterName, String keyword,
                                                 String namespace, String svcName, String workLoadName, int pageSize, int pageNo) throws Exception {

        return getK8sClusterPods(cloudVendor,k8sClusterName,
                keyword,namespace,svcName,
                workLoadName,null,pageSize, pageNo);
    }

    /**
     *
     * @param k8sClusterBasicModel
     * @param nameSpace
     * @param svcName
     * @return
     */
    @Override
    public SvcModel getSvcModelByName(K8sClusterBasicModel k8sClusterBasicModel, String nameSpace, String svcName) {

        SvcModel svcModel = new SvcModel();

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        if (k8sClient.services().inNamespace(nameSpace).withName(svcName) == null){
            return svcModel;
        }

        io.fabric8.kubernetes.api.model.Service svc
                = k8sClient.services().inNamespace(nameSpace).withName(svcName).get();


        if (svc == null){
            return svcModel;
        }

        svcModel.setSvcName(svc.getMetadata().getName());

        Map<String, String> selectorMap = svc.getSpec().getSelector();

        List<KeyValueModel> seclectorList = new ArrayList<>();
        if (selectorMap != null){
            for (String key:selectorMap.keySet()){
                seclectorList.add(new KeyValueModel(key,selectorMap.get(key)));
            }
            svcModel.setSeclectorMap(selectorMap);
        }else {
            svcModel.setSeclectorMap(new HashMap<>());
        }
        svcModel.setSeclectorList(seclectorList);

        return svcModel;
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param namespace
     * @param podName
     * @throws Exception
     */
    @Override
    public void delK8sClusterPod(String cloudVendor, String k8sClusterName, String namespace, String podName) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        k8sClient.pods().inNamespace(namespace).withName(podName).delete();
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param namespace
     * @param podName
     * @return
     * @throws Exception
     */
    @Override
    public List<ContainerModel> getK8sClusterPodContainers(String cloudVendor, String k8sClusterName, String namespace, String podName) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        List<PodModel> podModels = getAllPodsInfo(k8sClusterBasicModel,namespace,podName);
        for (PodModel podModel:podModels){
            if (StringUtils.equals(podModel.getPodName(),podName)){
                return podModel.getContainerModels();
            }
        }

        return new ArrayList<>();
    }

    @Override
    public RepByPage<SvcModel> getK8sClusterSvcs(String cloudVendor, String k8sClusterName, String keyword, int pageSize, int pageNo, String accountName) throws Exception {
        List<K8sClusterBasicModel> k8sClusterBasicModels
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName,accountName);
        if ((k8sClusterBasicModels == null)||(k8sClusterBasicModels.isEmpty())){
            RepByPage<SvcModel> result = new RepByPage<>();
            result.setResult(new ArrayList<>());
            result.setTotalNum(0);
            result.setPageNo(pageNo);
            result.setPageSize(pageSize);

            return result;
        }

        List<SvcModel> allSvcModels = new ArrayList<>();
        for (K8sClusterBasicModel k8sClusterBasicModel:k8sClusterBasicModels) {
            allSvcModels.addAll(getAllSvcInfo(k8sClusterBasicModel, null));
        }
        if (StringUtils.isNotEmpty(keyword)){

            List<SvcModel> svcModelsByKeyWord = new ArrayList<>();
            for (SvcModel svcModel:allSvcModels){
                if (StringUtils.contains(svcModel.getSvcName(),keyword)
                        ||StringUtils.contains(svcModel.getK8sClusterIp(),keyword)
                        ||StringUtils.contains(svcModel.getPort(),keyword)
                        ||StringUtils.contains(svcModel.getSeclector(),keyword)
                        ||StringUtils.contains(svcModel.getNamespace(),keyword)
                        ||StringUtils.contains(svcModel.getSvcType(),keyword)
                        ||StringUtils.contains(svcModel.getStatus(),keyword)){

                    svcModelsByKeyWord.add(svcModel);
                }
            }
            return OtherUtils.getResultFromList(svcModelsByKeyWord,pageNo,pageSize);
        }

        return OtherUtils.getResultFromList(allSvcModels,pageNo,pageSize);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param keyword
     * @param pageSize
     * @param pageNo
     * @return
     * @throws Exception
     */
    @Override
    public RepByPage<SvcModel> getK8sClusterSvcs(String cloudVendor, String k8sClusterName, String keyword, int pageSize, int pageNo) throws Exception {
        return getK8sClusterSvcs(cloudVendor, k8sClusterName,keyword,pageSize,pageNo,null);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param yaml
     * @throws Exception
     */
    @Override
    public void addK8sClusterSvc(String cloudVendor, String k8sClusterName, String yaml) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        //防止和其他svc弄混，写包全路径
        io.fabric8.kubernetes.api.model.Service newService = null;
        if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){

            newService = new io.fabric8.kubernetes.api.model.Service();
            newService.setMetadata(new ObjectMeta());
        }else {
            newService =k8sClient.services().load(new ByteArrayInputStream(yaml.getBytes())).get();
        }

        if (StringUtils.isEmpty(newService.getMetadata().getNamespace())){
            newService.getMetadata().setNamespace(CnapCloudMgmtConstants.DEFAULT_NAMESPACE);
        }

        //做判重处理
        //增加对ut的判断
        if ((!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName()))
            &&(k8sClient.services().inNamespace(newService.getMetadata().getNamespace())
                .withName(newService.getMetadata().getName()).get() != null)) {
            throw new Exception("该服务已存在");
        }

        if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {
            k8sClient.services().inNamespace(newService.getMetadata().getNamespace()).resource(newService).createOrReplace();
        }
    }

    /**
     * 
     * @param cloudVendor
     * @param k8sClusterName
     * @param namespace
     * @param svcType
     * @param svcName
     * @param port
     * @param targetPort
     * @param nodePort
     * @param selectorModels
     * @throws Exception
     */
    @Override
    public void editK8sClusterSvc(String cloudVendor, String k8sClusterName, String namespace,
                                  String svcType, String svcName, int port, int targetPort,
                                  int nodePort, List<KeyValueModel> selectorModels) throws Exception {


        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        ServicePort servicePort = new ServicePort();
        IntOrString targetIntOrStr = new IntOrString();
        targetIntOrStr.setValue(targetPort);
        servicePort.setTargetPort(targetIntOrStr);
        if ((nodePort != 0)
                &&(!StringUtils.equals(svcType,"ClusterIP"))) {
            servicePort.setNodePort(nodePort);
        }
        servicePort.setPort(port);

        Map<String,String> selectorMap = new HashMap<>();
        for (KeyValueModel keyValueModel:selectorModels){
            selectorMap.put(keyValueModel.getKey(),keyValueModel.getValue());
        }

        io.fabric8.kubernetes.api.model.Service createdSvc
                = new ServiceBuilder()
                .withNewMetadata()
                .withName(svcName)
                .endMetadata()
                .withNewSpec()
                .withPorts(servicePort)
                .addToSelector(selectorMap)
                .withType(svcType)
                .endSpec()
                .build();

        if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {
            k8sClient.services().inNamespace(namespace).resource(createdSvc).createOrReplace();
        }
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param namespace
     * @param svcName
     * @throws Exception
     */
    @Override
    public void delK8sClusterSvc(String cloudVendor, String k8sClusterName, String namespace, String svcName) throws Exception {
        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        k8sClient.services().inNamespace(namespace).withName(svcName).delete();
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param namespace
     * @param svcName
     * @param yaml
     * @throws Exception
     */
    @Override
    public void editK8sClusterSvcByYaml(String cloudVendor, String k8sClusterName, String namespace, String svcName, String yaml) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        KubernetesClient k8sClient = k8sClusterBasicModel.getK8sClient();

        KubernetesResource kubernetesResource = null;
        if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){
            io.fabric8.kubernetes.api.model.Service svc = new io.fabric8.kubernetes.api.model.Service();
            kubernetesResource = svc;
        }else {
            kubernetesResource = Serialization.unmarshal(yaml);
        }

        if (kubernetesResource instanceof io.fabric8.kubernetes.api.model.Service){
            io.fabric8.kubernetes.api.model.Service createdSvc =
                    (io.fabric8.kubernetes.api.model.Service)kubernetesResource;

            if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {
                k8sClient.services().inNamespace(namespace).resource(createdSvc).replace();
            }

        }
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param svcMonitorModels
     * @return
     * @throws Exception
     */
    @Override
    public List<SvcMonitorModel> getSvcMonitorInfo(String cloudVendor, String k8sClusterName, List<SvcMonitorModel> svcMonitorModels) throws Exception {


        List<SvcMonitorModel> result = new ArrayList<>();

        K8sClusterBasicModel k8sClusterBasicModel
                = multiCloudMgmtService.getK8sClientByCloudVendorAndName(cloudVendor,k8sClusterName);

        List<PodModel> allPodModels = getAllPodsInfo(k8sClusterBasicModel,null);

        //按pod的label来存放podmodel
        Map<String,List<PodModel>> podMapByLabel = new HashMap<>();
        for (PodModel podModel:allPodModels){

            for (String labelKey:podModel.getPodLabelMap().keySet()){

                String key = labelKey + podModel.getPodLabelMap().get(labelKey);
                if (!podMapByLabel.containsKey(key)){
                    podMapByLabel.put(key,new ArrayList<>());

                }
                podMapByLabel.get(key).add(podModel);
            }
        }

        for (SvcMonitorModel svcMonitorModel:svcMonitorModels){

            for (String labelKey:svcMonitorModel.getSeclectorMap().keySet()){

                String key = labelKey + svcMonitorModel.getSeclectorMap().get(labelKey);
                if (podMapByLabel.containsKey(key)){
                    for (PodModel podModel:podMapByLabel.get(key)){

                        if (svcMonitorModel.getTotalPodMap() == null){
                            svcMonitorModel.setTotalPodMap(new HashMap<>());
                        }
                        String podKey =
                                podModel.getPodIP() + podModel.getNamespace() + podModel.getPodName();
                        svcMonitorModel.getTotalPodMap().put(podKey,podKey);

                        if (StringUtils.equals(podModel.getStatus(),CnapCloudMgmtConstants.POD_STATUS_RUNNING)){
                            if (svcMonitorModel.getNormalPodMap() == null){
                                svcMonitorModel.setNormalPodMap(new HashMap<>());
                            }
                            svcMonitorModel.getNormalPodMap().put(podKey,podKey);
                        }
                    }

                    if (svcMonitorModel.getNormalPodMap() != null) {
                        svcMonitorModel.setNormalPodNum(svcMonitorModel.getNormalPodMap().size());
                    }else {
                        svcMonitorModel.setNormalPodNum(0);
                    }
                    if (svcMonitorModel.getTotalPodMap() != null) {
                        svcMonitorModel.setTotalPodNum(svcMonitorModel.getTotalPodMap().size());
                    }else {
                        svcMonitorModel.setTotalPodNum(0);
                    }
                }
            }
            result.add(svcMonitorModel);
        }
        
        return result;
    }

    /**
     * 
     * @param k8sClusterName
     * @param userName
     * @param beginTime
     * @param reason
     * @param pageSize
     * @param pageNo
     * @return
     */
    @Override
    public RepByPage<EventModel> getK8sClusterEvents(String k8sClusterName, String userName, String beginTime, String reason, int pageSize, int pageNo,String endTime) {

        return getK8sClusterEvents(k8sClusterName,userName,beginTime,reason,pageSize,pageNo,endTime,null);

    }

    /**
     *
     * @param k8sClusterName
     * @param userName
     * @param beginTime
     * @param reason
     * @param pageSize
     * @param pageNo
     * @return
     */
    @Override
    public RepByPage<EventModel> getK8sClusterEvents(String k8sClusterName, String userName, String beginTime, String reason, int pageSize, int pageNo) {

        return getK8sClusterEvents(k8sClusterName,userName,beginTime,reason,pageSize,pageNo,null);
    }

    @Override
    public RepByPage<EventModel> getK8sClusterEvents(String k8sClusterName, String userName, String beginTime,
                                                     String reason, int pageSize, int pageNo, String endTime,
                                                     String cloudVendor) {

        RepByPage<EventModel> result = new RepByPage<>();
        result.setPageNo(pageNo);
        result.setPageSize(pageSize);


        String k8sClusterIp = null;
        if (StringUtils.isNotEmpty(k8sClusterName)) {
            k8sClusterIp = multiCloudMgmtService.getK8sClusterIpByName(k8sClusterName);
        }

        LocalDateTime beginDate = null;
        final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        if(StringUtils.isNotEmpty(beginTime)) {
            beginDate = LocalDateTime.parse(beginTime, dateTimeFormatter);
        }

        LocalDateTime endDate = null;
        if(StringUtils.isNotEmpty(endTime)) {
            endDate = LocalDateTime.parse(endTime, dateTimeFormatter);
        }


        int totalNum = k8sClusterEventsMapper.countK8sClusterEvents(k8sClusterIp,userName,beginDate,reason,endDate,cloudVendor);
        result.setTotalNum(totalNum);
        if (totalNum > 0){

            result.setResult(k8sClusterEventsMapper
                    .getK8sClusterEventsByPage(k8sClusterIp,userName,beginDate,reason,pageSize,
                            (pageNo - 1)*pageSize,endDate,cloudVendor));
        }

        return result;
    }

    /**
     *
     * @param k8sClusterName
     * @param namespace
     * @param svcName
     * @return
     * @throws ParseException
     */
    @Override
    public SvcMonitorModel getSvcPodSpark(String k8sClusterName, String namespace, String svcName) throws ParseException {
        
        SvcMonitorModel svcMonitorModel = new SvcMonitorModel();
        Map<String,String> totalPodMap = new HashMap<>();
        Map<String,String> normalPodMap = new HashMap<>();

        K8sClusterBasicModel k8sClusterBasicModel = multiCloudMgmtService.getK8sClusterInfoByName(k8sClusterName);
        if (k8sClusterBasicModel == null){
            return svcMonitorModel;
        }

        //获取这个svc的情况
        SvcModel svcModel = null;
        if(StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){
            //for ut
            svcModel = new SvcModel();
            //set svcname
            svcModel.setSvcName("svcName");
            //for lbael selector
            svcModel.setSeclectorMap(new HashMap<>());
            //for label
            svcModel.getSeclectorMap().put("app","app");
        }else {
            svcModel = getSvcModelByName(k8sClusterBasicModel, namespace, svcName);
        }
        svcMonitorModel.setNamespace(namespace);
        svcMonitorModel.setK8sClusterName(k8sClusterName);
        svcMonitorModel.setSvcName(svcName);

        if (StringUtils.isEmpty(svcModel.getSvcName())){
            return svcMonitorModel;
        }

        //拿到所有符合这个selector的pod
        List<PodModel> podModels = null;
        if(StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {
            //for ut
            podModels = new ArrayList<>();
            //podmodel
            PodModel podModel = new PodModel();
            podModels.add(podModel);
            //podname
            podModel.setPodName(CnapCloudMgmtConstants.SPARK_PREFIX);
            //pod label
            podModel.setPodLabelMap(new HashMap<>());
            //pod label
            podModel.getPodLabelMap().put("app","app");
            //pod status
            podModel.setStatus(CnapCloudMgmtConstants.POD_STATUS_RUNNING);
            //set worker
            podModel.setWorkLoadModel(new WorkLoadModel());
            //woker name
            podModel.getWorkLoadModel().setWorkLoadName("workName");

        }else {
            podModels = getAllPodsInfo(k8sClusterBasicModel, namespace);

        }
        List<PodModel> podsForCount = new ArrayList<>();
        for (PodModel podModel:podModels){

            if ((podModel.getPodLabelMap() == null)||(podModel.getPodLabelMap().isEmpty())){
                continue;
            }

            for (String labelKey:podModel.getPodLabelMap().keySet()){

                if ((svcModel.getSeclectorMap().containsKey(labelKey))
                        &&(StringUtils.equals(svcModel.getSeclectorMap().get(labelKey),
                        podModel.getPodLabelMap().get(labelKey)))){

                    podsForCount.add(podModel);

                    totalPodMap.put(podModel.getPodName(),podModel.getPodName());
                    if (StringUtils.equals(podModel.getStatus(),CnapCloudMgmtConstants.POD_STATUS_RUNNING)){
                        normalPodMap.put(podModel.getPodName(),podModel.getPodName());
                    }
                }
            }
        }

        //根据找到的pod，反推得到work，然后查查work的情况,获取env里面固定前缀的
        String envPrefix = null;
        for (PodModel podModel:podsForCount){
            if (podModel.getWorkLoadModel() != null){

                Deployment deployment = null;
                if(StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,
                        k8sClusterBasicModel.getK8sClusterName())) {

                    //for ut;
                    deployment = new Deployment();
                    //set spec
                    deployment.setSpec(new DeploymentSpec());
                    //set template
                    deployment.getSpec().setTemplate(new PodTemplateSpec());
                    //set spec
                    deployment.getSpec().getTemplate().setSpec(new PodSpec());
                    //set contarner
                    deployment.getSpec().getTemplate().getSpec().setContainers(new ArrayList<>());
                    //container
                    Container container = new Container();
                    //env
                    container.setEnv(new ArrayList<>());
                    //env spark prefix
                    EnvVar var = new EnvVar();
                    container.getEnv().add(var);
                    var.setName(CnapCloudMgmtConstants.SPARK_PREFIX);
                    var.setValue(CnapCloudMgmtConstants.SPARK_PREFIX);
                }else{
                    deployment = k8sClusterBasicModel.getK8sClient()
                            .apps().deployments().inNamespace(namespace)
                            .withName(podModel.getWorkLoadModel().getWorkLoadName()).get();
                }

                //找到里面的容器的env
                List<Container> containers = deployment.getSpec().getTemplate().getSpec().getContainers();
                for (Container container:containers){

                    if (container.getEnv() != null){
                        for (EnvVar envVar:container.getEnv()){

                            if (StringUtils.equals(envVar.getName(),CnapCloudMgmtConstants.SPARK_PREFIX)
                                    && StringUtils.isNotEmpty(envVar.getValue())){
                                envPrefix = envVar.getValue();
                                break;
                            }
                        }
                    }

                    if (StringUtils.isNotEmpty(envPrefix)){
                        break;
                    }
                }
            }

            if (StringUtils.isNotEmpty(envPrefix)){
                break;
            }
        }

        //根据这个前缀，去找所有的pod
        Map<String,List<PodModel>> workLoadCountMap = new HashMap<>();
        List<String> statefulNames = new ArrayList<>();
        for (PodModel podModel:podModels){

            if (podModel.getWorkLoadModel() != null) {
                
                if (workLoadCountMap.containsKey(podModel.getWorkLoadModel().getWorkLoadName())) {

                    workLoadCountMap.get(podModel.getWorkLoadModel().getWorkLoadName()).add(podModel);

                } else {

                    workLoadCountMap.put(podModel.getWorkLoadModel().getWorkLoadName(), new ArrayList<>());
                    workLoadCountMap.get(podModel.getWorkLoadModel().getWorkLoadName()).add(podModel);
                }
            }

            if (StringUtils.contains(podModel.getPodName(),envPrefix)){
                //获取pod的label，然后找到这个label对应的statuful
                if (podModel.getPodLabelMap().containsKey(CnapCloudMgmtConstants.SPARK_POD_LABEL)){

                    statefulNames.add(
                            podModel.getPodLabelMap().get(CnapCloudMgmtConstants.SPARK_POD_LABEL));
                }

                totalPodMap.put(podModel.getPodName(),podModel.getPodName());
                if (StringUtils.equals(podModel.getStatus(),CnapCloudMgmtConstants.POD_STATUS_RUNNING)){
                    normalPodMap.put(podModel.getPodName(),podModel.getPodName());
                }
            }
        }

        //做最后的统计
        int otherPodNum = 0;
        if (!statefulNames.isEmpty()){

            for (String statefulName:statefulNames){

                for (String workloadName:workLoadCountMap.keySet()){
                    if (workloadName.contains(statefulName)){
                        otherPodNum += workLoadCountMap.get(workloadName).size();

                        //统计这些pod状态
                        for (PodModel podModel:workLoadCountMap.get(workloadName)){

                            totalPodMap.put(podModel.getPodName(),podModel.getPodName());
                            if (StringUtils.equals(podModel.getStatus(),CnapCloudMgmtConstants.POD_STATUS_RUNNING)){
                                normalPodMap.put(podModel.getPodName(),podModel.getPodName());
                            }
                        }
                    }
                }
            }
        }

        svcMonitorModel.setNormalPodNum(normalPodMap.size());
        svcMonitorModel.setTotalPodNum(totalPodMap.size());
        svcMonitorModel.setOtherPodNum(otherPodNum);

        return svcMonitorModel;
    }

    /**
     *
     * @param k8sClusterName
     * @param namespace
     * @param svcName
     * @return
     * @throws ParseException
     */
    @Override
    public SvcMonitorModel getSvcPodFlinks(String k8sClusterName, String namespace, String svcName) throws ParseException {

        SvcMonitorModel svcMonitorModel = new SvcMonitorModel();

        K8sClusterBasicModel k8sClusterBasicModel = multiCloudMgmtService.getK8sClusterInfoByName(k8sClusterName);
        if (k8sClusterBasicModel == null){
            return svcMonitorModel;
        }

        //获取这个svc的情况
        SvcModel svcModel = null;
        if(StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {

            //for ut
            svcModel = new SvcModel();
            //svcname
            svcModel.setSvcName("svcName");
            //svc labels
            svcModel.setSeclectorMap(new HashMap<>());
            //label kep
            svcModel.getSeclectorMap().put("app","app");
        }else {
            svcModel = getSvcModelByName(k8sClusterBasicModel,namespace,svcName);
        }
            svcMonitorModel.setNamespace(namespace);
        svcMonitorModel.setK8sClusterName(k8sClusterName);
        svcMonitorModel.setSvcName(svcName);

        if (StringUtils.isEmpty(svcModel.getSvcName())){
            return svcMonitorModel;
        }

        //拿到所有符合这个selector的pod
        List<PodModel> podModels = null;
        if(StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {
           //for ut
            podModels = new ArrayList<>();
            PodModel podModel = new PodModel();
            //for label map
            podModel.setPodLabelMap(new HashMap<>());
            //for label key
            podModel.getPodLabelMap().put("app","app");
            //wokerlaod
            podModel.setWorkLoadModel(new WorkLoadModel());
            podModel.getWorkLoadModel().setWorkLoadName(CnapCloudMgmtConstants.FLINK_PREFIX);
            //podname
            podModel.setPodName("podName");
            //status
            podModel.setStatus( CnapCloudMgmtConstants.POD_STATUS_RUNNING);
            podModels.add(podModel);

        }else {
            podModels = getAllPodsInfo(k8sClusterBasicModel, namespace);
        }
        List<PodModel> podsForCount = new ArrayList<>();
        Map<String,PodModel> result = new HashMap<>();
        String envPrefix = null;
        for (PodModel podModel:podModels){

            if ((podModel.getPodLabelMap() == null)||(podModel.getPodLabelMap().isEmpty())){
                continue;
            }

            for (String labelKey:podModel.getPodLabelMap().keySet()){

                if ((svcModel.getSeclectorMap().containsKey(labelKey))
                        &&(StringUtils.equals(svcModel.getSeclectorMap().get(labelKey),
                                                podModel.getPodLabelMap().get(labelKey)))){

                    podsForCount.add(podModel);
                    result.put(podModel.getPodName(),podModel);
                }
            }
        }

        //根据找到的pod，反推得到work，然后查查work的情况,获取env里面固定前缀的
        for (PodModel podModel:podsForCount){
            if (podModel.getWorkLoadModel() != null){
                Deployment deployment = null;
                if(StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {

                    //for ut
                    deployment = new Deployment();
                    //for spec
                    deployment.setSpec(new DeploymentSpec());
                    //for temaplate
                    deployment.getSpec().setTemplate(new PodTemplateSpec());
                    //spec
                    deployment.getSpec().getTemplate().setSpec(new PodSpec());
                    //for containers
                    deployment.getSpec().getTemplate().getSpec().setContainers(new ArrayList<>());
                    //container
                    Container container = new Container();
                    deployment.getSpec().getTemplate().getSpec().getContainers().add(container);
                    //set en
                    container.setEnv(new ArrayList<>());
                    EnvVar envVar = new EnvVar();
                    container.getEnv().add(envVar);
                    //设置名称
                    envVar.setName(CnapCloudMgmtConstants.FLINK_PREFIX);
                    envVar.setValue(CnapCloudMgmtConstants.FLINK_PREFIX);
                }else {

                    deployment = k8sClusterBasicModel.getK8sClient()
                            .apps().deployments().inNamespace(namespace)
                            .withName(podModel.getWorkLoadModel().getWorkLoadName()).get();
                }
                
                //找到里面的容器的env
                List<Container> containers = deployment.getSpec().getTemplate().getSpec().getContainers();
                for (Container container:containers){

                    if (container.getEnv() != null){
                        for (EnvVar envVar:container.getEnv()){

                            if (StringUtils.equals(envVar.getName(),CnapCloudMgmtConstants.FLINK_PREFIX)
                                    && StringUtils.isNotEmpty(envVar.getValue())){
                                envPrefix = envVar.getValue();
                                break;
                            }
                        }
                    }

                    if (StringUtils.isNotEmpty(envPrefix)){
                        break;
                    }
                }
            }

            if (StringUtils.isNotEmpty(envPrefix)){
                break;
            }
        }

        //根据找到的envPreFix去找depolyment,之前所有的namespace下的pod已经获取了
        if (StringUtils.isNotEmpty(envPrefix)) {
            for (PodModel podModel : podModels) {

                if ((podModel.getWorkLoadModel() != null)
                        && (StringUtils.contains(podModel.getWorkLoadModel().getWorkLoadName(), envPrefix))) {

                    result.put(podModel.getPodName(), podModel);

                }

            }
        }

        //做最后的统计
        int podTotalNum = 0;
        int podNormalNum = 0;
        for (String podName:result.keySet()){

            if (StringUtils.equals(result.get(podName).getStatus(),
                    CnapCloudMgmtConstants.POD_STATUS_RUNNING)){
                podNormalNum++;
            }
            podTotalNum++;
        }

        svcMonitorModel.setNormalPodNum(podNormalNum);
        svcMonitorModel.setTotalPodNum(podTotalNum);

        return svcMonitorModel;
    }

    /**
     *
     * @param k8sClient
     * @return
     */
    @Override
    public String getK8sClusterInnerIp(KubernetesClient k8sClient) {

        /**
         * for ut
         * k8sClient调用不可能为null
         * 所以使用这个作为ut判断
         */
        List<Node> nodes = null;
        if (k8sClient == null) {
           nodes = new ArrayList<>();
           Node node = new Node();
           nodes.add(node);
           //setStatus
           node.setStatus(new NodeStatus());
           //setConditions
           node.getStatus().setConditions(new ArrayList<>());
           //node status
            NodeCondition nodeCondition = new NodeCondition();
            node.getStatus().getConditions().add(nodeCondition);
            //setType
            nodeCondition.setType("Ready");
            //setstats
            nodeCondition.setStatus("True");
            //set master ip
            node.setMetadata(new ObjectMeta());
            //set labels
            node.getMetadata().setLabels(new HashMap<>());
            //set master label
            node.getMetadata().getLabels().put("node-role.kubernetes.io/master","node-role.kubernetes.io/master");
            //set address
            node.getStatus().setAddresses(new ArrayList<>());
            //node address
            NodeAddress nodeAddress = new NodeAddress();
            nodeAddress.setType("IP");
            nodeAddress.setAddress(CnapCloudMgmtConstants.TEST_ADMIN);
            //setnode adderss
            node.getStatus().getAddresses().add(nodeAddress);
        }else {
            //正常情况从client获取
            nodes = k8sClient.nodes().list().getItems();
        }

        for (Node node:nodes) {

            String nodeStatus = null;
            if (node.getStatus().getConditions() != null) {
                for (NodeCondition nodeCondition : node.getStatus().getConditions()) {
                    if (StringUtils.equals(nodeCondition.getType(), "Ready")) {
                        if (!StringUtils.equals(nodeCondition.getStatus(), "True")) {

                            nodeStatus = K8sStatusEnum.Error.getDescribe();
                            break;
                        }
                    }
                }
            }

            if (StringUtils.equals(nodeStatus,K8sStatusEnum.Error.getDescribe())){
                continue;
            }

            //只看master节点的ip
            if (!node.getMetadata().getLabels().containsKey("node-role.kubernetes.io/master")) {
                continue;
            }

            //ip
            List<NodeAddress> nodeAddresseList = node.getStatus().getAddresses();
            for (NodeAddress nodeAddress : nodeAddresseList) {

                if (StringUtils.contains(nodeAddress.getType(), "IP")) {
                    return nodeAddress.getAddress();

                }
            }
        }

        return null;
    }

    /**
     *
     * @param k8sClusterName
     * @param nodeName
     * @param labelKey
     * @throws Exception
     */
    @Override
    public void delNodeLabel(String k8sClusterName,String nodeName,String labelKey) throws Exception {

        //记录加节点操作
        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("K8sClusterServiceImpl delNodeLabel clustername={} nodeName={} labelKey={}",
                k8sClusterName,nodeName,labelKey);

        K8sClusterBasicModel k8sClusterBasicModel =
                multiCloudMgmtService.getK8sClusterInfoByName(k8sClusterName);

        if (!StringUtils.equals(k8sClusterBasicModel.getK8sStatus(),K8sStatusEnum.Running.getChineseDescribe())){
            LOGGER.warn("K8sClusterServiceImpl delNodeLabel cluster status error clustername={} nodeName={}",
                    k8sClusterBasicModel.getK8sClusterName(),nodeName);
            return;
        }
        if ((!StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),CnapCloudMgmtConstants.TEST_ADMIN))
                &&(k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName) == null)) {
            throw new Exception("node name not exist");
        }
        if ((!StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),CnapCloudMgmtConstants.TEST_ADMIN))
                &&((k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName) == null)
                ||(k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName).get() == null))){
            throw new Exception("node " + nodeName + " is null");
        }

        Node node = null;
        if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){
            node = new Node();
            node.setMetadata(new ObjectMeta());
            node.getMetadata().setLabels(null);
        }else {
            //获取节点
            node = k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName).get();
        }

        if (node.getMetadata() == null){
            LOGGER.warn("K8sClusterServiceImpl addNodeLabel node.metada is null ip={} nodeName={}",
                    k8sClusterBasicModel.getK8sClusterIp(),nodeName);
            return;
        }

        Map<String, String> labelMap = node.getMetadata().getLabels();
        if (labelMap == null){
            return;
        }

        labelMap.remove(labelKey);

        node.getMetadata().setLabels(labelMap);

        if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){
            k8sClusterBasicModel.getK8sClient().nodes().resource(node).createOrReplace();
        }
    }


    /**
     *
     * @param k8sClusterBasicModel
     * @param nodeName
     * @param labelKey
     * @param labelValue
     * @throws Exception
     */
    @Override
    public void addNodeLabel(K8sClusterBasicModel k8sClusterBasicModel,
                             String nodeName, String labelKey, String labelValue) throws Exception {

        //记录加节点操作
        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("K8sClusterServiceImpl addNodeLabel ip={} nodeName={} labelKey={} labeValue={}",
                k8sClusterBasicModel.getK8sClusterIp(),nodeName,labelKey,labelValue);
        
        if (!StringUtils.equals(k8sClusterBasicModel.getK8sStatus(),K8sStatusEnum.Running.getChineseDescribe())){
            LOGGER.warn("K8sClusterServiceImpl addNodeLabel cluster status error ip={} nodeName={}",
                    k8sClusterBasicModel.getK8sClusterIp(),nodeName);
            return;
        }
        if ((!StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),CnapCloudMgmtConstants.TEST_ADMIN))
                &&(k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName) == null)) {
            throw new Exception("node name not exist");
        }
        if ((!StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),CnapCloudMgmtConstants.TEST_ADMIN))
                        &&((k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName) == null)
                ||(k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName).get() == null))){
            throw new Exception("node " + nodeName + " is null");
        }

        if (StringUtils.isEmpty(labelValue)){
            LOGGER.error("K8sClusterServiceImpl addNodeLabel ip={} nodeName={} labelKey={} labeValue={} labeValue is empty",
                    k8sClusterBasicModel.getK8sClusterIp(),nodeName,labelKey,labelValue);
            return;
        }


        labelValue = labelValue.toLowerCase();
        Node node = null;
        if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){
            node = new Node();
            node.setMetadata(new ObjectMeta());
            node.getMetadata().setLabels(null);
        }else {

            node = k8sClusterBasicModel.getK8sClient().nodes().withName(nodeName).get();
        }

        if (node == null){
            LOGGER.warn("K8sClusterServiceImpl addNodeLabel node is null ip={} nodeName={}",
                    k8sClusterBasicModel.getK8sClusterIp(),nodeName);
            return;
        }

        if (node.getMetadata() == null){
            LOGGER.warn("K8sClusterServiceImpl addNodeLabel node.metada is null ip={} nodeName={}",
                    k8sClusterBasicModel.getK8sClusterIp(),nodeName);
            return;
        }

        Map<String, String> labelMap = node.getMetadata().getLabels();
        if (labelMap == null){
            labelMap = new HashMap<>();
        }

        labelMap.put(labelKey,labelValue);

        node.getMetadata().setLabels(labelMap);

        if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){
            k8sClusterBasicModel.getK8sClient().nodes().resource(node).createOrReplace();
        }
    }

    @Override
    public void changeNodeLabel(String k8sClusterName, String nodeName, String labelKey, String labelValue) throws Exception {
        if (StringUtils.isEmpty(labelValue)){
            return;
        }

        K8sClusterBasicModel k8sClusterBasicModel = multiCloudMgmtService.getK8sClusterInfoByName(k8sClusterName);
        labelValue = labelValue.toLowerCase();
        addNodeLabel(k8sClusterBasicModel,nodeName,labelKey,labelValue);
    }

    @Override
    public void addNodeLabel(String k8sClusterName, String nodeName, String labelKey, String labelValue) throws Exception {

        if (StringUtils.isEmpty(labelValue)){
            return;
        }

        K8sClusterBasicModel k8sClusterBasicModel = multiCloudMgmtService.getK8sClusterInfoByName(k8sClusterName);
        labelValue = labelValue.toLowerCase();
        addNodeLabel(k8sClusterBasicModel,nodeName,labelKey,labelValue);
    }


    /**
     *
     * @param nodeModel
     */
    @Override
    public void addK8sClusterNode(NodeModel nodeModel) {
    }


    @Override
    public K8sClusterBasicModel installCnapScheduleAgent(String k8sClusterIp,String arch) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel = multiCloudMgmtService.getK8sClientByIp(k8sClusterIp);

        List<EnvVar> envs = new ArrayList<>();
        EnvVar adminAddress = new EnvVar();
        adminAddress.setName("ADMIN_ADDRESS");
        adminAddress.setValue(agentAdminUrl);
        envs.add(adminAddress);

        EnvVar clientAddress = new EnvVar();
        clientAddress.setName("CLIENT_ADDRESS");
        clientAddress.setValue("http://" + k8sClusterIp + ":30286");
        envs.add(clientAddress);

        EnvVar clusterName = new EnvVar();
        clusterName.setName("CLUSTER_NAME");
        clusterName.setValue(k8sClusterBasicModel.getK8sClusterName());
        envs.add(clusterName);

        EnvVar config = new EnvVar();
        config.setName("KUBE_CONFIG_PATH");
        config.setValue("/root/.kube/config");
        envs.add(config);

        VolumeMount initContainerVolumMount = new VolumeMount();
        initContainerVolumMount.setName("kubeconfig");
        initContainerVolumMount.setMountPath("/root/.kube/config");

        Volume volume = new Volume();
        volume.setName("kubeconfig");
        HostPathVolumeSource hostPath = new HostPathVolumeSource();
        hostPath.setPath("/root/.kube/config");
        volume.setHostPath(hostPath);

        Map<String,String> nodeSelectorMap = new HashMap<>();
        nodeSelectorMap.put("node-type","master");

        Deployment newDeployment = new DeploymentBuilder()
                .withNewMetadata()
                    .withName("cnap-schedule-agent")
                    .addToLabels("app","cnap-schedule-agent")
                .endMetadata()
                .withNewSpec()
                    .withReplicas(1)
                    .withNewTemplate()
                        .withNewMetadata()
                            .addToLabels("app", "cnap-schedule-agent")
                        .endMetadata()
                        .withNewSpec()
                            .withVolumes(volume)
                            .withNodeSelector(nodeSelectorMap)
                            .addNewContainer()
                                .withName("cnap-schedule-agent")
                                .withImage(agentImage)
                                .withEnv(envs)
                                .withVolumeMounts(initContainerVolumMount)
                            .endContainer()
                        .endSpec()
                    .endTemplate()
                .withNewSelector()
                    .addToMatchLabels("app","cnap-schedule-agent")
                .endSelector()
                .endSpec()
                .build();

        io.fabric8.kubernetes.api.model.Service agentSvc = null;
        if (StringUtils.equalsIgnoreCase(arch,CnapCloudMgmtConstants.ARCH_GPU)) {
            Map<String, String> agentSvcSelectorMap = new HashMap<>();
            agentSvcSelectorMap.put("app", "cnap-schedule-agent");

            io.fabric8.kubernetes.api.model.ServicePort svcPort = new ServicePort();
            svcPort.setProtocol("TCP");
            svcPort.setPort(8080);
            IntOrString targetPort = new IntOrString(8080);
            svcPort.setTargetPort(targetPort);
            svcPort.setNodePort(30286);

            agentSvc = new ServiceBuilder()
                    .withNewMetadata()
                    .withName("cnap-schedule-agent-svc")
                    .endMetadata()
                    .withNewSpec()
                    .withType("NodePort")
                    .withSelector(agentSvcSelectorMap)
                    .withPorts(svcPort)
                    .endSpec().build();

        }


        if (!StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())) {
            k8sClusterBasicModel.getK8sClient().apps().deployments().inNamespace("default")
                    .resource(newDeployment).createOrReplace();
            if (agentSvc != null) {
                k8sClusterBasicModel.getK8sClient().services().inNamespace("default")
                        .resource(agentSvc).createOrReplace();
            }
        }

        return k8sClusterBasicModel;

    }

    /**
     *
     * @param K8sClusterIp
     * @return
     */
    @Override
    public PodModel getUtListPods(String K8sClusterIp) throws Exception {
        PodModel resultPodModel = new PodModel();
        K8sClusterBasicModel k8sClusterBasicModel =
        multiCloudMgmtService.getK8sClientByIp(K8sClusterIp);
        //reutn result
        List<PodModel> podModels = null;
        if (StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),
                CnapCloudMgmtConstants.TEST_ADMIN)){

            podModels = new ArrayList<>();
            PodModel podModel = new PodModel();
            PodModel podModelNo = new PodModel();
            podModelNo.setStatus(K8sStatusEnum.Error.getDescribe());
            podModels.add(podModelNo);
            podModels.add(podModel);
            podModel.setStatus(K8sStatusEnum.Running.getDescribe());
            podModel.setPodLabelMap(new HashMap<>());
            podModel.getPodLabelMap().put(CnapCloudMgmtConstants.TEST_ADMIN,"app");
            WorkLoadModel workLoadModel = new WorkLoadModel();
            podModel.setWorkLoadModel(workLoadModel);
            workLoadModel.setWorkLoadKind(CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DEPLOYMENT);
        }else {
             podModels = getAllPodsInfo(k8sClusterBasicModel,"ns");
        }

        long cpuUse=0;
        long memUse = 0;
        long podLimitCpuByN = 0;
        long memLimit = 0;
        if(podModels != null){
            for (PodModel podModel:podModels){
                if (!StringUtils.equals(podModel.getStatus(),
                        K8sStatusEnum.Running.getDescribe())){
                    continue;
                }

                cpuUse += podModel.getCpuUseByN();
                memUse +=podModel.getMemUseByKi();
                podLimitCpuByN +=podModel.getPodLimitCpuByN();
                memLimit += podModel.getPodLimitMemByKi();

                if ((podModel.getPodLabelMap() != null)
                       && (podModel.getPodLabelMap().containsKey(CnapCloudMgmtConstants.TEST_ADMIN))){
                    resultPodModel.setStatus(K8sStatusEnum.Ready.getDescribe());
                }
                EventModel eventModel = new EventModel();
                eventModel.setCloudVendor(k8sClusterBasicModel.getCloudVendor());
                eventModel.setK8sClusterIp(k8sClusterBasicModel.getK8sClusterIp());
                eventModel.setEventType(CnapCloudMgmtConstants.TEST_ADMIN);
                eventModel.setMsg(podModel.getStatus());
                eventModel.setNamespace("ns");
                eventModel.setInvolvedObj(PodModel.class.toString());
                k8sClusterEventsMapper.addK8sClusterEvent(eventModel);

                k8sClusterInfoMapper.delK8sClusterByIp(k8sClusterBasicModel.getK8sClusterIp());

                k8sClusterInfoMapper.updateClusterVmInfo(k8sClusterBasicModel.getK8sClusterIp(),
                        JSONObject.toJSONString(podModel));
                if (podModel.getWorkLoadModel()!=null){

                    if (StringUtils.equals(podModel.getWorkLoadModel().getWorkLoadKind(),
                            CnapCloudMgmtConstants.CNAP_WORKLOAD_KIND_DEPLOYMENT)){
                        resultPodModel.setWorkLoadModel(podModel.getWorkLoadModel());
                    }
                }
            }
        }

        resultPodModel.setCpuUseByN(cpuUse);
        resultPodModel.setMemUseByKi(memUse);
        resultPodModel.setPodLimitCpuByN(podLimitCpuByN);
        resultPodModel.setPodLimitMemByKi(memLimit);
        resultPodModel.setCloudVendor(k8sClusterBasicModel.getCloudVendor());
        resultPodModel.setK8sClusterName(k8sClusterBasicModel.getK8sClusterName());
        resultPodModel.setVpcId(k8sClusterBasicModel.getVpcId());
        if (cacheService.getVmCreateTime(String.valueOf(resultPodModel.getCpuUseByN())) < 0L){
            multiCloudMgmtService.getAllK8sClusterIps();
        }
        return resultPodModel;
    }

    /**
     *
     * @param k8sClusterIp
     * @return
     * @throws Exception
     */
    @Override
    public boolean checkSvcExist(String k8sClusterIp) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel = multiCloudMgmtService.getK8sClientByIp(k8sClusterIp);
        //如果集群失联，表明可以删除
        if (!StringUtils.equals(k8sClusterBasicModel.getK8sStatus(), K8sStatusEnum.Running.getChineseDescribe())) {
            LOGGER.error("checkSvcExist k8sCluster status error ip={}", k8sClusterIp);
            return false;
        }

        List<PodModel> podModels = new ArrayList<>();
        //构造ut
        if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,k8sClusterBasicModel.getK8sClusterName())){
            PodModel podModel = new PodModel();
            podModel.setNamespace("app-teset");
            podModel.setStatus(CnapCloudMgmtConstants.POD_STATUS_RUNNING);
            podModels.add(podModel);
        }else {
            //获取所有pod
            podModels = getAllPodsInfo(k8sClusterBasicModel, null);
        }
        for (PodModel podModel : podModels) {
            //如果ns开头是app，且运行状态，就认为不能删除
            if (StringUtils.containsIgnoreCase(podModel.getNamespace(), "app")
                    && (StringUtils.equals(podModel.getStatus(), CnapCloudMgmtConstants.POD_STATUS_RUNNING))){
                return true;
            }
        }

        return false;
    }

    @Override
    public boolean checkSvcSuspend(K8sClusterBasicModel k8sClusterBasicModel, String svcId) {

        try {
            List<SvcModel> svcModels = getAllSvcInfo(k8sClusterBasicModel, null);
            if ((svcModels == null)||(svcModels.isEmpty())){
                return true;
            }

            LOGGER.error("checkSvcSuspend param svcId={} user={}",svcId,k8sClusterBasicModel.getCreateUser());


            MicroServicePO microServicePO =
                    microSvcMapper.getMicroServiceByUserNameSvcName(k8sClusterBasicModel.getCreateUser(),null,svcId);
            for (SvcModel svcModel:svcModels){

                //svc存在，说明没有暂停
                if (StringUtils.equalsIgnoreCase(svcModel.getSvcName(),microServicePO.getServiceName())){
                    return false;
                }
            }

        }catch (Exception e){
            LOGGER.error("checkSvcSuspend failed svcId={}",svcId);
            return false;
        }

        return true;
    }

    @Override
    public void refeshK8sClusterInfos(String k8sClusterIp) throws Exception {


        //保存所有最近5分钟发生了请求的微服务的id
        Set<String> microSvcIdSet = new HashSet<>();

        //k8sCluseterIp不为null说明只是刷新集群信息的
        if (StringUtils.isEmpty(k8sClusterIp)) {
            //拿到所有微服务的信息,处理之前先做清除
            cacheService.cleanAllMicroSvcCache();
            List<MicroServicePO> microServicePOS = microSvcMapper.getAllMicroService(null, null);
            if (!microServicePOS.isEmpty()) {
                long nowTimestamp = (new Date()).getTime();
                for (MicroServicePO microServicePO : microServicePOS) {
                    cacheService.putMicroSvc(microServicePO.getUserName(), microServicePO);

                    //计算最近5分钟这个服务相关的请求时间，如果存在就处理
                    String microSvcId = microServicePO.getId();
                    List<GatewayLogModel> gatewayLogModels
                            = microSvcMapper.getGatewayLogByTime(microSvcId, new Date(nowTimestamp - 5 * 60 * 1000));
                    if ((gatewayLogModels != null) && (!gatewayLogModels.isEmpty())) {

                        microSvcIdSet.add(microSvcId);

                        int logNum = gatewayLogModels.size();
                        int reqTotalTime = 0;
                        for (GatewayLogModel gatewayLogModel : gatewayLogModels) {
                            reqTotalTime += gatewayLogModel.getReqCost();
                        }
                        MicroSvcReqCostInfoModel microSvcReqCostInfoModel = new MicroSvcReqCostInfoModel();
                        microSvcReqCostInfoModel.setId(cn.hutool.core.lang.UUID.randomUUID().toString(true));
                        microSvcReqCostInfoModel.setCreateTime(new Date());
                        microSvcReqCostInfoModel.setReqCost(reqTotalTime / logNum);
                        microSvcReqCostInfoModel.setMicroSvcName(microServicePO.getMicroServiceName());
                        microSvcReqCostInfoModel.setMicroSvcId(microSvcId);
                        microSvcUseInfoMapper.saveMicroReqCostInfo(microSvcReqCostInfoModel);
                    }
                }
            }
        }

        /*
         *先获取所有集群的信息
         * 设置一个比较大的分页，用于获取所有
         */
        String k8sClusterName = null;
        if (StringUtils.isNotEmpty(k8sClusterIp)){
            k8sClusterName = multiCloudMgmtService.getK8sClientByIp(k8sClusterIp).getK8sClusterName();
        }
        RepByPage<K8sClusterModel> k8sClusterModelRepByPage
                = multiCloudMgmtService.getK8sClusterInfos(null,k8sClusterName,1000,1,null);
        if ((k8sClusterModelRepByPage.getResult() == null)||(k8sClusterModelRepByPage.getResult().isEmpty())){
            if (StringUtils.isEmpty(k8sClusterIp)) {
                cacheService.clearAllK8sInfoCache();
            }
            return;
        }

        //用于缓存的，给微服务做查找的辅助map
        Map<String,List<PodModel>> podMapForMicroSvc = new HashMap<>();
        Map<String,NodeModel> nodeModelMap = new HashMap<>();

        //记录label为empty节点，这种节点，如果部署了微服务，需要给他打上label用于别的服务处理
        //key 是集群ip加nodename
        Map<String,NodeModel> nodeEmptyLabelMap = new HashMap<>();

        //获取各种信息放入缓存
        Set<String> clusterNameSet = new HashSet<>();
        Date nowDate = new Date();
        for (K8sClusterModel k8sClusterModel:k8sClusterModelRepByPage.getResult()){

            //只有运行状态的才统计，没运行的给状态删除
            if (!StringUtils.equals(k8sClusterModel.getStatus(),K8sStatusEnum.Running.getChineseDescribe())){

                //不处于运行状态的，说明集群异常了，要记录告警
                //如果记录了集群告警，那就增加告警
                RulePO rulePO = alarmMapper.getClusterTimeOverAlarmRule(k8sClusterModel.getK8sClusterName());
                if (rulePO != null){
                    try {
                        //生成告警插入
                        insertAlarm(rulePO, k8sClusterModel);
                    }catch (Exception e){
                        LOGGER.warn(" insert alarm failed name={}",k8sClusterModel.getK8sClusterName(),e);
                    }
                }

                cacheService.clearK8sInfoCache(k8sClusterModel.getCreateUser(), k8sClusterModel.getK8sClusterName());
                continue;
            }

            clusterNameSet.add(k8sClusterModel.getK8sClusterName());
            k8sClusterModel.setResfeshDate(nowDate);

            //集群自身信息
            cacheService.putK8sClusterModels(k8sClusterModel.getCreateUser(),k8sClusterModel);

            //节点信息
            RepByPage<NodeModel> nodeModelRepByPage = getK8sClusterNodeInfosByConditions(
                    k8sClusterModel.getCloudVendor(),k8sClusterModel.getK8sClusterName(),null,1000,1);
            cacheService.putK8sNodeInfo(k8sClusterModel.getCreateUser(),nodeModelRepByPage);
            if (nodeModelRepByPage.getResult() != null){
                for (NodeModel nodeModel:nodeModelRepByPage.getResult()){

                    //保存节点的历史数据信息，
                    NodeHistoryInfoModel nodeHistoryInfoModel = new NodeHistoryInfoModel();
                    nodeHistoryInfoModel.setNodeName(nodeModel.getNodeName());
                    nodeHistoryInfoModel.setK8sClusterName(nodeModel.getK8sClusterName());
                    nodeHistoryInfoModel.setAccountName(nodeModel.getAccountName());
                    nodeHistoryInfoModel.setUserName(nodeModel.getCreateUser());
                    nodeHistoryInfoModel.setNodeSvcId(nodeModel.getNodeLabelMap().get(CnapCloudMgmtConstants.NODE_APP_LABEL));
                    nodeHistoryInfoModel.setNodeCpuQuotaM((int)(nodeModel.getNodeCpuQuotaByN()/(1000*1000)));
                    nodeHistoryInfoModel.setNodeCpuUseM((int)(nodeModel.getNodeCpuUseByN()/(1000*1000)));
                    nodeHistoryInfoModel.setNodeMemQuotaKi((int)nodeModel.getNodeMemQuotaByKi());
                    nodeHistoryInfoModel.setNodeMemUseKi((int)nodeModel.getNodeMemUseNumByKi());
                    nodeHistoryInfoModel.setNodeDiskQuootaGi(nodeModel.getNodeDiskQuota());
                    nodeHistoryInfoModel.setNodeDiskUseGi(nodeModel.getNodeDiskUseNum());
                    nodeHistoryInfoModel.setNodeGpuTemp(nodeModel.getNodeGpuTemp());
                    nodeHistoryInfoModel.setNodeGpuUsage(nodeModel.getNodeGpuUsage());
                    nodeHistoryInfoModel.setNodeGpuPower(nodeModel.getNodeGpuPower());
                    nodeHistoryInfoModel.setNodeGpuMemUseGi(nodeModel.getNodeGpuMemUse());
                    nodeHistoryInfoModel.setNodeGpuMemTotalGi(nodeModel.getNodeGpuMemTotal());
                    nodeHistoryInfoModel.setNodeGpuMemUsage(nodeModel.getNodeGpuMemUsage());
                    nodeHistoryInfoModel.setDiskIops(nodeModel.getDiskIops());
                    nodeHistoryInfoModel.setDiskThroghput(nodeModel.getDiskThroughput());
                    nodeHistoryInfoModel.setCreateDate(new Date());

                    microSvcUseInfoMapper.saveNodeHistoryInfo(nodeHistoryInfoModel);

                    nodeModelMap.put(nodeModel.getK8sClusterIp() + nodeModel.getNodeName(),nodeModel);

                        //看看是不是svc label已经消失
                        if (StringUtils.equals(
                                nodeModel.getNodeLabelMap().get(CnapCloudMgmtConstants.NODE_APP_LABEL),
                                CnapCloudMgmtConstants.NODE_APP_LABEL_EMPTY)){
                            nodeEmptyLabelMap.put(nodeModel.getK8sClusterIp()+nodeModel.getNodeName(),
                                    nodeModel);
                        }
                }
            }

            //namespace信息
            RepByPage<NameSpaceModel> nameSpaceModelRepByPage = getK8sClusterNameSpaces(k8sClusterModel.getCloudVendor(),
                    k8sClusterModel.getK8sClusterName(),null,1000,1);
            cacheService.putK8sNsInfo(k8sClusterModel.getCreateUser(),nameSpaceModelRepByPage);

            //workload信息
            RepByPage<WorkLoadModel> workLoadModelRepByPage = getK8sClusterWorkLoads(k8sClusterModel.getCloudVendor(),
                    k8sClusterModel.getK8sClusterName(),null,null,1000,1);
            cacheService.putK8sWorkLoadInfo(k8sClusterModel.getCreateUser(),workLoadModelRepByPage);

            //pod信息
            RepByPage<PodModel> podModelRepByPage = getK8sClusterPods(k8sClusterModel.getCloudVendor(),
                    k8sClusterModel.getK8sClusterName(),null,null,
                    null,null,1000,1);
            cacheService.putK8sPodInfo(k8sClusterModel.getCreateUser(),podModelRepByPage);
            for (PodModel podModel:podModelRepByPage.getResult()){

                if (podModel.getPodLabelMap() != null){

                    for (String labelKey:podModel.getPodLabelMap().keySet()){

                        //key用集群ip加label组成
                        String key = podModel.getK8sClusterIp()
                                + labelKey + podModel.getPodLabelMap().get(labelKey);
                        if (!podMapForMicroSvc.containsKey(key)){
                            podMapForMicroSvc.put(key,new ArrayList<>());
                        }
                        podMapForMicroSvc.get(key).add(podModel);
                    }
                }
            }

            //svc信息
            RepByPage<SvcModel> svcModelRepByPage = getK8sClusterSvcs(k8sClusterModel.getCloudVendor(),
                    k8sClusterModel.getK8sClusterName(),null,1000,1);
            cacheService.putK8sSvcInfo(k8sClusterModel.getCreateUser(),svcModelRepByPage);
        }

        //获取所有被缓存的集群，查查有没有被删除，被删除的就把他给clear掉
        if (StringUtils.isEmpty(k8sClusterIp)) {
            List<String> userNames = cacheService.getAllK8sUser();
            if (userNames.isEmpty()) {
                return;
            }
            for (String userName : userNames) {

                List<String> clusterNames = cacheService.getK8sClusterNames(userName);
                for (String clusterName : clusterNames) {
                    //不包含,说明集群已经被删了，要清除
                    if (!clusterNameSet.contains(clusterName)) {
                        cacheService.clearK8sInfoCache(userName, clusterName);
                    }
                }
            }
        }

        if (StringUtils.isEmpty(k8sClusterIp)) {
            //保存一个微服务历史的使用数据
            List<SvcModel> svcModels = cacheService.getSvcModelsForMicroSvc();
            if ((svcModels != null) && (!svcModels.isEmpty())) {
                for (SvcModel svcModel : svcModels) {

                    MicroSvcDataSaveModel microSvcDataSaveModel = new MicroSvcDataSaveModel();
                    microSvcDataSaveModel.setMicroSvcId(svcModel.getMicroServicePO().getId());
                    microSvcDataSaveModel.setSvcName(svcModel.getSvcName());
                    microSvcDataSaveModel.setRecordTime(new Date());
                    microSvcDataSaveModel.setMicroSvcName(svcModel.getMicroServiceName());
                    microSvcDataSaveModel.setK8sClusterIp(svcModel.getK8sClusterIp());
                    microSvcDataSaveModel.setK8sClusterName(svcModel.getK8sClusterName());

                    for (String selecterKey : svcModel.getSeclectorMap().keySet()) {

                        //svc对应的podmodels做处理
                        List<PodModel> podModels
                                = podMapForMicroSvc.get(svcModel.getK8sClusterIp() + selecterKey
                                + svcModel.getSeclectorMap().get(selecterKey));
                        if (podModels != null) {

                            //每个podModel存储一个
                            for (PodModel podModel : podModels) {

                                microSvcDataSaveModel.setPodName(podModel.getPodName());
                                microSvcDataSaveModel.setCpuUse(
                                        QuantityUtils.changeCpuNtoMInt(podModel.getCpuUseByN()));

                                NodeModel nodeModel = nodeModelMap.get(svcModel.getK8sClusterIp() + podModel.getNodeName());
                                if (nodeModel == null) {
                                    continue;
                                }

                                //如果这个node在空master的集合中，就把这个master节点给他加label
                                if (nodeEmptyLabelMap.containsKey(nodeModel.getK8sClusterIp() + nodeModel.getNodeName())) {
                                    LOGGER.warn("k8ClusterInfoScheduled add master label svcName={} svcId={}", svcModel.getSvcName(),
                                            svcModel.getMicroServicePO().getId());
                                    //装svc的label，用于选择亲和性
                                    addNodeLabel(nodeModel.getK8sClusterName(), nodeModel.getNodeName(),
                                            CnapCloudMgmtConstants.NODE_APP_LABEL,
                                            svcModel.getSvcName());

                                    addNodeLabel(nodeModel.getK8sClusterName(), nodeModel.getNodeName(),
                                            CnapCloudMgmtConstants.NODE_APP_LABEL_ID,
                                            svcModel.getMicroServicePO().getId());
                                }

                                //获取cpu使用量
                                if (podModel.getPodLimitCpuByN() != 0L) {
                                    microSvcDataSaveModel.setCpuTotal(
                                            QuantityUtils.changeCpuNtoMInt(podModel.getPodLimitCpuByN()));
                                } else {

                                    microSvcDataSaveModel.setCpuTotal(
                                            QuantityUtils.changeCpuNtoMInt(nodeModel.getNodeCpuQuotaByN()));
                                }

                                //获取mem使用量
                                microSvcDataSaveModel.setMemUse(
                                        QuantityUtils.changeMemKitoMiNoMi(podModel.getMemUseByKi())
                                );
                                if (podModel.getPodLimitMemByKi() != 0L) {
                                    microSvcDataSaveModel.setMemTotal(
                                            QuantityUtils.changeMemKitoMiNoMi(podModel.getPodLimitMemByKi()));
                                } else {
                                    microSvcDataSaveModel.setMemTotal(
                                            QuantityUtils.changeMemKitoMiNoMi(nodeModel.getNodeMemQuotaByKi()));
                                }

                                //保存disk之类数据
                                microSvcDataSaveModel.setDiskThroughput(nodeModel.getDiskThroughput());
                                microSvcDataSaveModel.setDiskIops(nodeModel.getDiskIops());
                                microSvcDataSaveModel.setNetBandwidth(nodeModel.getNetBandwidth());
                                if (microSvcIdSet.contains(svcModel.getMicroServicePO().getId())) {
                                    microSvcDataSaveModel.setReqFlag(1);
                                }

                                //如果是GPU的svc应用，那就保存nodel相关的gpu监控数据
                                if (StringUtils.endsWithIgnoreCase(microSvcDataSaveModel.getK8sClusterName(),
                                        CnapCloudMgmtConstants.ARCH_GPU)) {

                                    microSvcDataSaveModel.setGpuMemory(nodeModel.getNodeGpuMemUse());
                                    microSvcDataSaveModel.setGpuMemUsage(nodeModel.getNodeGpuMemUsage());
                                    microSvcDataSaveModel.setGpuTemp(nodeModel.getNodeGpuTemp());
                                    microSvcDataSaveModel.setGpuUsage(nodeModel.getNodeGpuUsage());
                                }

                                //保存这个历史数据
                                microSvcDataSaveModel.setId(UUID.randomUUID().toString(true));
                                microSvcUseInfoMapper.saveMicroSvcUseInfo(microSvcDataSaveModel);
                            }
                        }
                    }
                }
            }
        }
    }

    private void insertAlarm(RulePO rulePO,K8sClusterModel k8sClusterModel) throws UnsupportedEncodingException {

        RuleResourceInfo resourceInfo = rulePO.getResourceInfo();
        SingleResInfo singleResInfo = resourceInfo.getResInfos().get(0);

        // 生成告警po
        AlarmPO alarmPO = new AlarmPO();
        // 使用自己的时间，防止两边系统时间不一致
        alarmPO.setCreateTime(LocalDateTime.now());
        alarmPO.setResourceName(singleResInfo.getResName());
        alarmPO.setResourceType(resourceInfo.getResType());
        alarmPO.setDetailMsg(resourceInfo.getIndicatorName() + "over time");

        alarmPO.setRuleId(rulePO.getId());
        alarmPO.setId(IdUtil.simpleUUID());

        alarmPO.setClusterName(k8sClusterModel.getK8sClusterName());
        alarmPO.setProvider(k8sClusterModel.getCloudVendor());

        LOGGER.warn("notce alam alarm={}",JSON.toJSONString(alarmPO));
        alarmMapper.insertAlarmPO(alarmPO);
        //邮件通知
        MailNoticeAlarmModel noticeAlarmModel = new MailNoticeAlarmModel();
        noticeAlarmModel.setAlarmPOList(new ArrayList<>());
        noticeAlarmModel.getAlarmPOList().add(alarmPO);
        noticeAlarmModel.setRulePO(rulePO);
        String url = "http://cnap-monitor-svc.default.svc.cluster.local:8090" + "/sys/sendAlarmMail";
        doHttpPostReq(url,new StringEntity(JSON.toJSONString(noticeAlarmModel)));
    }

    /**
     *
     * @param url
     * @param param
     * @return
     */
    private String doHttpPostReq(String url, StringEntity param){

        try {

            CloseableHttpClient httpclient = HttpClients.createDefault();

            HttpPost httpPost = new HttpPost(url);

            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(5 * 60 * 1000).setConnectionRequestTimeout(5 * 60 * 1000)
                    .setSocketTimeout(5 * 60 * 1000).build();
            httpPost.setConfig(requestConfig);
            httpPost.setEntity(param);

            CloseableHttpResponse response = httpclient.execute(httpPost);
            return EntityUtils.toString(response.getEntity());

        }catch (Throwable e){
            LOGGER.error("insertAlarm doHttpPostReq fail url={}",url,e);
        }

        return null;
    }


    @Override
    public void stopErrorInject(ErrorInjectModel errorInjectModel) throws Exception {

        K8sClusterBasicModel k8sClusterBasicModel =
                multiCloudMgmtService.getK8sClusterInfoByName(errorInjectModel.getK8sClusterName());
        if (!StringUtils.equalsIgnoreCase(k8sClusterBasicModel.getK8sStatus(),
                K8sStatusEnum.Running.getChineseDescribe())){

            throw new Exception("集群 " + errorInjectModel.getK8sClusterName() + " 状态异常, 请稍后再试");
        }

        //找到要执行的节点
        for (VmModel vmModel:k8sClusterBasicModel.getVmModels()){
            //找到要执行的ip
            if (StringUtils.containsIgnoreCase(errorInjectModel.getNodeName(),vmModel.getInnerIp())){
                cnapSshService.stopErrorInject(vmModel,errorInjectModel.getRunningId());
                break;
            }
        }
    }

    @Override
    public String runErrorInject(ErrorInjectModel errorInjectModel) throws Exception {
        K8sClusterBasicModel k8sClusterBasicModel =
                multiCloudMgmtService.getK8sClusterInfoByName(errorInjectModel.getK8sClusterName());
        if (!StringUtils.equalsIgnoreCase(k8sClusterBasicModel.getK8sStatus(),
                K8sStatusEnum.Running.getChineseDescribe())){

            throw new Exception("集群 " + errorInjectModel.getK8sClusterName() + " 状态异常, 请稍后再试");
        }

        //找到要执行的节点
        for (VmModel vmModel:k8sClusterBasicModel.getVmModels()){
            //找到要执行的ip
            if (StringUtils.containsIgnoreCase(errorInjectModel.getNodeName(),vmModel.getInnerIp())){
                return cnapSshService.runErrorInject(vmModel,errorInjectModel.getErrorOrder());
            }
        }

        return null;
    }

    @Override
    public void microSvcPress(ErrorInjectModel errorInjectModel) throws ParseException {
        //拿到这个svcid对应的信息
        MicroServicePO microServicePO =
                microSvcMapper.getMicroServiceByUserNameId(
                        errorInjectModel.getUsername(),errorInjectModel.getMicroSvcId());
        //获取集群对应的ip，和这个微服务的端口
        K8sClusterBasicModel k8sClusterBasicModel =
                multiCloudMgmtService.getK8sClusterInfoByName(errorInjectModel.getK8sClusterName());
        List<SvcModel> svcModels =
                getAllSvcInfo(k8sClusterBasicModel,microServicePO.getNamespace());

        MicroSvcPressParamModel microSvcPressParamModel =
                JSONObject.parseObject(errorInjectModel.getErrorOrder(),MicroSvcPressParamModel.class);

        JmeterDTO jmeterDTO = new JmeterDTO();
        jmeterDTO.setDomain(k8sClusterBasicModel.getK8sClusterIp());
        jmeterDTO.setLoopCount(microSvcPressParamModel.getLoopTime());
        jmeterDTO.setRampUp(microSvcPressParamModel.getWaitTime());
        jmeterDTO.setThreadNum(microSvcPressParamModel.getAccountNum());
        jmeterDTO.setPath("/");
        if (StringUtils.containsIgnoreCase(microServicePO.getDeploymentName(),"face")){
            jmeterDTO.setSvcType("face");
        }else {
            jmeterDTO.setSvcType("helmet");
        }
        for (SvcModel svcModel:svcModels){
            if (StringUtils.equalsIgnoreCase(svcModel.getSvcName(),microServicePO.getServiceName())){
                jmeterDTO.setPath(svcModel.getNodePort());
            }
        }

        jmeterService.start(jmeterDTO);
    }
}
